From de43e2a8e7b36cf5e5c4e5c3f5185c85e5e03dea Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 20:05:38 +0000 Subject: [PATCH 001/230] merge fallout --- .../main/java/io/questdb/client/Sender.java | 199 ++- .../cutlass/http/client/WebSocketClient.java | 775 +++++++++ .../http/client/WebSocketClientFactory.java | 137 ++ .../http/client/WebSocketClientLinux.java | 71 + .../http/client/WebSocketClientOsx.java | 75 + .../http/client/WebSocketClientWindows.java | 74 + .../http/client/WebSocketFrameHandler.java | 93 ++ .../http/client/WebSocketSendBuffer.java | 582 +++++++ .../ilpv4/client/GlobalSymbolDictionary.java | 173 ++ .../cutlass/ilpv4/client/IlpBufferWriter.java | 177 ++ .../ilpv4/client/IlpV4WebSocketEncoder.java | 790 +++++++++ .../ilpv4/client/IlpV4WebSocketSender.java | 1398 ++++++++++++++++ .../cutlass/ilpv4/client/InFlightWindow.java | 468 ++++++ .../ilpv4/client/MicrobatchBuffer.java | 501 ++++++ .../ilpv4/client/NativeBufferWriter.java | 289 ++++ .../cutlass/ilpv4/client/ResponseReader.java | 247 +++ .../ilpv4/client/WebSocketChannel.java | 668 ++++++++ .../ilpv4/client/WebSocketResponse.java | 283 ++++ .../ilpv4/client/WebSocketSendQueue.java | 693 ++++++++ .../ilpv4/protocol/IlpV4BitReader.java | 335 ++++ .../ilpv4/protocol/IlpV4BitWriter.java | 247 +++ .../ilpv4/protocol/IlpV4ColumnDef.java | 163 ++ .../ilpv4/protocol/IlpV4Constants.java | 506 ++++++ .../ilpv4/protocol/IlpV4GorillaDecoder.java | 251 +++ .../ilpv4/protocol/IlpV4GorillaEncoder.java | 235 +++ .../ilpv4/protocol/IlpV4NullBitmap.java | 310 ++++ .../ilpv4/protocol/IlpV4SchemaHash.java | 574 +++++++ .../ilpv4/protocol/IlpV4TableBuffer.java | 1424 +++++++++++++++++ .../ilpv4/protocol/IlpV4TimestampDecoder.java | 474 ++++++ .../cutlass/ilpv4/protocol/IlpV4Varint.java | 261 +++ .../cutlass/ilpv4/protocol/IlpV4ZigZag.java | 98 ++ .../ilpv4/websocket/WebSocketCloseCode.java | 178 +++ .../ilpv4/websocket/WebSocketFrameParser.java | 342 ++++ .../ilpv4/websocket/WebSocketFrameWriter.java | 281 ++++ .../ilpv4/websocket/WebSocketHandshake.java | 421 +++++ .../ilpv4/websocket/WebSocketOpcode.java | 136 ++ .../client/std/CharSequenceIntHashMap.java | 209 +++ .../io/questdb/client/std/LongHashSet.java | 155 ++ core/src/main/java/module-info.java | 3 + 39 files changed, 14292 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java create mode 100644 core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java create mode 100644 core/src/main/java/io/questdb/client/std/LongHashSet.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index c63609f..fcc6d46 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -34,6 +34,7 @@ import io.questdb.client.cutlass.line.http.AbstractLineHttpSender; import io.questdb.client.cutlass.line.tcp.DelegatingTlsChannel; import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; +import io.questdb.client.cutlass.ilpv4.client.IlpV4WebSocketSender; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; @@ -144,7 +145,21 @@ static LineSenderBuilder builder(CharSequence configurationString) { * @return Builder object to create a new Sender instance. */ static LineSenderBuilder builder(Transport transport) { - return new LineSenderBuilder(transport == Transport.HTTP ? LineSenderBuilder.PROTOCOL_HTTP : LineSenderBuilder.PROTOCOL_TCP); + int protocol; + switch (transport) { + case HTTP: + protocol = LineSenderBuilder.PROTOCOL_HTTP; + break; + case TCP: + protocol = LineSenderBuilder.PROTOCOL_TCP; + break; + case WEBSOCKET: + protocol = LineSenderBuilder.PROTOCOL_WEBSOCKET; + break; + default: + throw new LineSenderException("unknown transport: " + transport); + } + return new LineSenderBuilder(protocol); } /** @@ -461,7 +476,15 @@ enum Transport { * and for use-cases where HTTP transport is not suitable, when communicating with a QuestDB server over a high-latency * network */ - TCP + TCP, + + /** + * Use WebSocket transport to communicate with a QuestDB server. + *

+ * WebSocket transport uses the ILP v4 binary protocol for efficient data ingestion. + * It supports both synchronous and asynchronous modes with flow control. + */ + WEBSOCKET } /** @@ -522,6 +545,13 @@ final class LineSenderBuilder { private static final int PARAMETER_NOT_SET_EXPLICITLY = -1; private static final int PROTOCOL_HTTP = 1; private static final int PROTOCOL_TCP = 0; + private static final int PROTOCOL_WEBSOCKET = 2; + private static final int DEFAULT_WEBSOCKET_PORT = 9000; + private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8; + private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16; + private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500; + private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB + private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; @@ -568,6 +598,11 @@ public int getTimeout() { private char[] trustStorePassword; private String trustStorePath; private String username; + // WebSocket-specific fields + private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY; + private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY; + private boolean asyncMode = false; + private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY; private LineSenderBuilder() { @@ -733,6 +768,47 @@ public LineSenderBuilder autoFlushRows(int autoFlushRows) { return this; } + /** + * Set the maximum number of bytes per batch before auto-flushing. + *
+ * This is only used when communicating over WebSocket transport. + *
+ * Default value is 1MB. + * + * @param bytes maximum bytes per batch + * @return this instance for method chaining + */ + public LineSenderBuilder autoFlushBytes(int bytes) { + if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush bytes was already configured") + .put("[bytes=").put(this.autoFlushBytes).put("]"); + } + if (bytes < 0) { + throw new LineSenderException("auto flush bytes cannot be negative") + .put("[bytes=").put(bytes).put("]"); + } + this.autoFlushBytes = bytes; + return this; + } + + /** + * Enable asynchronous mode for WebSocket transport. + *
+ * In async mode, rows are batched and sent asynchronously with flow control. + * This provides higher throughput at the cost of more complex error handling. + *
+ * This is only used when communicating over WebSocket transport. + *
+ * Default is synchronous mode (false). + * + * @param enabled whether to enable async mode + * @return this instance for method chaining + */ + public LineSenderBuilder asyncMode(boolean enabled) { + this.asyncMode = enabled; + return this; + } + /** * Configure capacity of an internal buffer. *

@@ -791,6 +867,39 @@ public Sender build() { username, password, maxNameLength, actualMaxRetriesNanos, maxBackoffMillis, actualMinRequestThroughput, actualAutoFlushIntervalMillis, protocolVersion); } + if (protocol == PROTOCOL_WEBSOCKET) { + if (hosts.size() != 1 || ports.size() != 1) { + throw new LineSenderException("only a single address (host:port) is supported for WebSocket transport"); + } + + int actualAutoFlushRows = autoFlushRows == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_ROWS : autoFlushRows; + int actualAutoFlushBytes = autoFlushBytes == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_BYTES : autoFlushBytes; + long actualAutoFlushIntervalNanos = autoFlushIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY + ? DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS + : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis); + int actualInFlightWindowSize = inFlightWindowSize == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_IN_FLIGHT_WINDOW_SIZE : inFlightWindowSize; + int actualSendQueueCapacity = sendQueueCapacity == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEND_QUEUE_CAPACITY : sendQueueCapacity; + + if (asyncMode) { + return IlpV4WebSocketSender.connectAsync( + hosts.getQuick(0), + ports.getQuick(0), + tlsEnabled, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + actualInFlightWindowSize, + actualSendQueueCapacity + ); + } else { + return IlpV4WebSocketSender.connect( + hosts.getQuick(0), + ports.getQuick(0), + tlsEnabled + ); + } + } + assert protocol == PROTOCOL_TCP; if (hosts.size() != 1 || ports.size() != 1) { @@ -1048,6 +1157,29 @@ public LineSenderBuilder httpUsernamePassword(String username, String password) return this; } + /** + * Set the maximum number of batches that can be in-flight awaiting server acknowledgment. + *
+ * This is only used when communicating over WebSocket transport with async mode enabled. + *
+ * Default value is 8. + * + * @param size maximum number of in-flight batches + * @return this instance for method chaining + */ + public LineSenderBuilder inFlightWindowSize(int size) { + if (this.inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("in-flight window size was already configured") + .put("[size=").put(this.inFlightWindowSize).put("]"); + } + if (size < 1) { + throw new LineSenderException("in-flight window size must be positive") + .put("[size=").put(size).put("]"); + } + this.inFlightWindowSize = size; + return this; + } + /** * Configures the maximum backoff time between retry attempts when the Sender encounters recoverable errors. *
@@ -1239,6 +1371,29 @@ public LineSenderBuilder retryTimeoutMillis(int retryTimeoutMillis) { return this; } + /** + * Set the capacity of the send queue for batches waiting to be sent. + *
+ * This is only used when communicating over WebSocket transport with async mode enabled. + *
+ * Default value is 16. + * + * @param capacity send queue capacity + * @return this instance for method chaining + */ + public LineSenderBuilder sendQueueCapacity(int capacity) { + if (this.sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("send queue capacity was already configured") + .put("[capacity=").put(this.sendQueueCapacity).put("]"); + } + if (capacity < 1) { + throw new LineSenderException("send queue capacity must be positive") + .put("[capacity=").put(capacity).put("]"); + } + this.sendQueueCapacity = capacity; + return this; + } + private static int getValue(CharSequence configurationString, int pos, StringSink sink, String name) { if ((pos = ConfStringParser.value(configurationString, pos, sink)) < 0) { throw new LineSenderException("invalid ").put(name).put(" [error=").put(sink).put("]"); @@ -1275,7 +1430,13 @@ private void configureDefaults() { maximumBufferCapacity = protocol == PROTOCOL_HTTP ? DEFAULT_MAXIMUM_BUFFER_CAPACITY : bufferCapacity; } if (ports.size() == 0) { - ports.add(protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT : DEFAULT_TCP_PORT); + if (protocol == PROTOCOL_HTTP) { + ports.add(DEFAULT_HTTP_PORT); + } else if (protocol == PROTOCOL_WEBSOCKET) { + ports.add(DEFAULT_WEBSOCKET_PORT); + } else { + ports.add(DEFAULT_TCP_PORT); + } } if (tlsValidationMode == null) { tlsValidationMode = TlsValidationMode.DEFAULT; @@ -1334,8 +1495,16 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else if (Chars.equals("tcps", sink)) { tcp(); tlsEnabled = true; + } else if (Chars.equals("ws", sink)) { + if (tlsEnabled) { + throw new LineSenderException("cannot use ws protocol when TLS is enabled. use wss instead"); + } + websocket(); + } else if (Chars.equals("wss", sink)) { + websocket(); + tlsEnabled = true; } else { - throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps]]"); + throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps, ws, wss]]"); } String tcpToken = null; @@ -1557,6 +1726,14 @@ private void tcp() { protocol = PROTOCOL_TCP; } + private void websocket() { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol was already configured ") + .put("[protocol=").put(protocol).put("]"); + } + protocol = PROTOCOL_WEBSOCKET; + } + private void validateParameters() { if (hosts.size() == 0) { throw new LineSenderException("questdb server address not set"); @@ -1617,6 +1794,20 @@ private void validateParameters() { if (autoFlushIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("auto flush interval is not supported for TCP protocol"); } + } else if (protocol == PROTOCOL_WEBSOCKET) { + if (privateKey != null) { + throw new LineSenderException("TCP authentication is not supported for WebSocket protocol"); + } + if (httpToken != null || username != null || password != null) { + // TODO: WebSocket auth not yet implemented + throw new LineSenderException("Authentication is not yet supported for WebSocket protocol"); + } + if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { + throw new LineSenderException("in-flight window size requires async mode"); + } + if (sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { + throw new LineSenderException("send queue capacity requires async mode"); + } } else { throw new LineSenderException("unsupported protocol ") .put("[protocol=").put(protocol).put("]"); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java new file mode 100644 index 0000000..8f5ce83 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -0,0 +1,775 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Misc; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; +import io.questdb.client.std.str.Utf8String; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +/** + * Zero-GC WebSocket client built on QuestDB's native socket infrastructure. + *

+ * This client uses native memory buffers and non-blocking I/O with + * platform-specific event notification (epoll/kqueue/select). + *

+ * Features: + *

+ *

+ * Thread safety: This class is NOT thread-safe. Each connection should be + * accessed from a single thread at a time. + */ +public abstract class WebSocketClient implements QuietCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class); + + private static final int DEFAULT_RECV_BUFFER_SIZE = 65536; + private static final int DEFAULT_SEND_BUFFER_SIZE = 65536; + + protected final NetworkFacade nf; + protected final Socket socket; + + private final WebSocketSendBuffer sendBuffer; + private final WebSocketFrameParser frameParser; + private final Rnd rnd; + private final int defaultTimeout; + + // Receive buffer (native memory) + private long recvBufPtr; + private int recvBufSize; + private int recvPos; // Write position + private int recvReadPos; // Read position + + // Connection state + private CharSequence host; + private int port; + private boolean upgraded; + private boolean closed; + + // Handshake key for verification + private String handshakeKey; + + public WebSocketClient(HttpClientConfiguration configuration, SocketFactory socketFactory) { + this.nf = configuration.getNetworkFacade(); + this.socket = socketFactory.newInstance(nf, LOG); + this.defaultTimeout = configuration.getTimeout(); + + int sendBufSize = Math.max(configuration.getInitialRequestBufferSize(), DEFAULT_SEND_BUFFER_SIZE); + int maxSendBufSize = Math.max(configuration.getMaximumRequestBufferSize(), sendBufSize); + this.sendBuffer = new WebSocketSendBuffer(sendBufSize, maxSendBufSize); + + this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE); + this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT); + this.recvPos = 0; + this.recvReadPos = 0; + + this.frameParser = new WebSocketFrameParser(); + this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.upgraded = false; + this.closed = false; + } + + @Override + public void close() { + if (!closed) { + closed = true; + + // Try to send close frame + if (upgraded && !socket.isClosed()) { + try { + sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null, 1000); + } catch (Exception e) { + // Ignore errors during close + } + } + + disconnect(); + sendBuffer.close(); + + if (recvBufPtr != 0) { + Unsafe.free(recvBufPtr, recvBufSize, MemoryTag.NATIVE_DEFAULT); + recvBufPtr = 0; + } + } + } + + /** + * Disconnects the socket without closing the client. + * The client can be reconnected by calling connect() again. + */ + public void disconnect() { + Misc.free(socket); + upgraded = false; + host = null; + port = 0; + recvPos = 0; + recvReadPos = 0; + } + + /** + * Connects to a WebSocket server. + * + * @param host the server hostname + * @param port the server port + * @param timeout connection timeout in milliseconds + */ + public void connect(CharSequence host, int port, int timeout) { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + + // Close existing connection if connecting to different host:port + if (this.host != null && (!this.host.equals(host) || this.port != port)) { + disconnect(); + } + + if (socket.isClosed()) { + doConnect(host, port, timeout); + } + + this.host = host; + this.port = port; + } + + /** + * Connects using default timeout. + */ + public void connect(CharSequence host, int port) { + connect(host, port, defaultTimeout); + } + + private void doConnect(CharSequence host, int port, int timeout) { + int fd = nf.socketTcp(true); + if (fd < 0) { + throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']'); + } + + if (nf.setTcpNoDelay(fd, true) < 0) { + LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno()); + } + + socket.of(fd); + nf.configureKeepAlive(fd); + + long addrInfo = nf.getAddrInfo(host, port); + if (addrInfo == -1) { + disconnect(); + throw new HttpClientException("could not resolve host [host=").put(host).put(']'); + } + + if (nf.connectAddrInfo(fd, addrInfo) != 0) { + int errno = nf.errno(); + nf.freeAddrInfo(addrInfo); + disconnect(); + throw new HttpClientException("could not connect [host=").put(host) + .put(", port=").put(port) + .put(", errno=").put(errno).put(']'); + } + nf.freeAddrInfo(addrInfo); + + if (nf.configureNonBlocking(fd) < 0) { + int errno = nf.errno(); + disconnect(); + throw new HttpClientException("could not configure non-blocking [fd=").put(fd) + .put(", errno=").put(errno).put(']'); + } + + if (socket.supportsTls()) { + try { + socket.startTlsSession(host); + } catch (TlsSessionInitFailedException e) { + int errno = nf.errno(); + disconnect(); + throw new HttpClientException("could not start TLS session [fd=").put(fd) + .put(", error=").put(e.getFlyweightMessage()) + .put(", errno=").put(errno).put(']'); + } + } + + setupIoWait(); + LOG.debug("Connected to [host={}, port={}]", host, port); + } + + /** + * Performs WebSocket upgrade handshake. + * + * @param path the WebSocket endpoint path (e.g., "/ws") + * @param timeout timeout in milliseconds + */ + public void upgrade(CharSequence path, int timeout) { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + if (socket.isClosed()) { + throw new HttpClientException("Not connected"); + } + if (upgraded) { + return; // Already upgraded + } + + // Generate random key + byte[] keyBytes = new byte[16]; + for (int i = 0; i < 16; i++) { + keyBytes[i] = (byte) rnd.nextInt(256); + } + handshakeKey = Base64.getEncoder().encodeToString(keyBytes); + + // Build upgrade request + sendBuffer.reset(); + sendBuffer.putAscii("GET "); + sendBuffer.putAscii(path); + sendBuffer.putAscii(" HTTP/1.1\r\n"); + sendBuffer.putAscii("Host: "); + sendBuffer.putAscii(host); + if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) { + sendBuffer.putAscii(":"); + sendBuffer.putAscii(Integer.toString(port)); + } + sendBuffer.putAscii("\r\n"); + sendBuffer.putAscii("Upgrade: websocket\r\n"); + sendBuffer.putAscii("Connection: Upgrade\r\n"); + sendBuffer.putAscii("Sec-WebSocket-Key: "); + sendBuffer.putAscii(handshakeKey); + sendBuffer.putAscii("\r\n"); + sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n"); + sendBuffer.putAscii("\r\n"); + + // Send request + long startTime = System.nanoTime(); + doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout); + + // Read response + int remainingTimeout = remainingTime(timeout, startTime); + readUpgradeResponse(remainingTimeout); + + upgraded = true; + sendBuffer.reset(); + LOG.debug("WebSocket upgraded [path={}]", path); + } + + /** + * Performs upgrade with default timeout. + */ + public void upgrade(CharSequence path) { + upgrade(path, defaultTimeout); + } + + private void readUpgradeResponse(int timeout) { + // Read HTTP response into receive buffer + long startTime = System.nanoTime(); + + while (true) { + int remainingTimeout = remainingTime(timeout, startTime); + int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout); + if (bytesRead > 0) { + recvPos += bytesRead; + } + + // Check for end of headers (\r\n\r\n) + int headerEnd = findHeaderEnd(); + if (headerEnd > 0) { + validateUpgradeResponse(headerEnd); + // Compact buffer - move remaining data to start + int remaining = recvPos - headerEnd; + if (remaining > 0) { + Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining); + } + recvPos = remaining; + recvReadPos = 0; + return; + } + + if (recvPos >= recvBufSize) { + throw new HttpClientException("HTTP response too large"); + } + } + } + + private int findHeaderEnd() { + // Look for \r\n\r\n + for (int i = 0; i < recvPos - 3; i++) { + if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') { + return i + 4; + } + } + return -1; + } + + private void validateUpgradeResponse(int headerEnd) { + // Extract response as string for parsing + byte[] responseBytes = new byte[headerEnd]; + for (int i = 0; i < headerEnd; i++) { + responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i); + } + String response = new String(responseBytes, StandardCharsets.US_ASCII); + + // Check status line + if (!response.startsWith("HTTP/1.1 101")) { + String statusLine = response.split("\r\n")[0]; + throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine); + } + + // Verify Sec-WebSocket-Accept + String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey); + if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) { + throw new HttpClientException("Invalid Sec-WebSocket-Accept header"); + } + } + + // === Sending === + + /** + * Gets the send buffer for building WebSocket frames. + *

+ * Usage: + * 

+     * WebSocketSendBuffer buf = client.getSendBuffer();
+     * buf.beginBinaryFrame();
+     * buf.putLong(data);
+     * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+     * client.sendFrame(frame, timeout);
+     * buf.reset();
+     *
+ */ + public WebSocketSendBuffer getSendBuffer() { + return sendBuffer; + } + + /** + * Sends a complete WebSocket frame. + * + * @param frame frame info from endBinaryFrame() + * @param timeout timeout in milliseconds + */ + public void sendFrame(WebSocketSendBuffer.FrameInfo frame, int timeout) { + checkConnected(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + } + + /** + * Sends a complete WebSocket frame with default timeout. + */ + public void sendFrame(WebSocketSendBuffer.FrameInfo frame) { + sendFrame(frame, defaultTimeout); + } + + /** + * Sends binary data as a WebSocket binary frame. + * + * @param dataPtr pointer to data + * @param length data length + * @param timeout timeout in milliseconds + */ + public void sendBinary(long dataPtr, int length, int timeout) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginBinaryFrame(); + sendBuffer.putBlockOfBytes(dataPtr, length); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + + /** + * Sends binary data with default timeout. + */ + public void sendBinary(long dataPtr, int length) { + sendBinary(dataPtr, length, defaultTimeout); + } + + /** + * Sends a ping frame. + */ + public void sendPing(int timeout) { + checkConnected(); + sendBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePingFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + + /** + * Sends a close frame. + */ + public void sendCloseFrame(int code, String reason, int timeout) { + sendBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason); + try { + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + } finally { + sendBuffer.reset(); + } + } + + private void doSend(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + while (len > 0) { + int remainingTimeout = remainingTime(timeout, startTime); + ioWait(remainingTimeout, IOOperation.WRITE); + int sent = dieIfNegative(socket.send(ptr, len)); + while (socket.wantsTlsWrite()) { + remainingTimeout = remainingTime(timeout, startTime); + ioWait(remainingTimeout, IOOperation.WRITE); + dieIfNegative(socket.tlsIO(Socket.WRITE_FLAG)); + } + if (sent > 0) { + ptr += sent; + len -= sent; + } + } + } + + // === Receiving === + + /** + * Receives and processes WebSocket frames. + * + * @param handler frame handler callback + * @param timeout timeout in milliseconds + * @return true if a frame was received, false on timeout + */ + public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) { + checkConnected(); + + // First, try to parse any data already in buffer + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + + // Need more data + long startTime = System.nanoTime(); + while (true) { + int remainingTimeout = remainingTime(timeout, startTime); + if (remainingTimeout <= 0) { + return false; // Timeout + } + + // Ensure buffer has space + if (recvPos >= recvBufSize - 1024) { + growRecvBuffer(); + } + + int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout); + if (bytesRead <= 0) { + return false; // Timeout + } + recvPos += bytesRead; + + result = tryParseFrame(handler); + if (result != null) { + return result; + } + } + } + + /** + * Receives frame with default timeout. + */ + public boolean receiveFrame(WebSocketFrameHandler handler) { + return receiveFrame(handler, defaultTimeout); + } + + /** + * Non-blocking attempt to receive a WebSocket frame. + * Returns immediately if no complete frame is available. + * + * @param handler frame handler callback + * @return true if a frame was received, false if no data available + */ + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + checkConnected(); + + // First, try to parse any data already in buffer + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + + // Try one non-blocking recv + if (recvPos >= recvBufSize - 1024) { + growRecvBuffer(); + } + + int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + if (n == 0) { + return false; // No data available + } + recvPos += n; + + // Try to parse again + result = tryParseFrame(handler); + return result != null && result; + } + + private Boolean tryParseFrame(WebSocketFrameHandler handler) { + if (recvPos <= recvReadPos) { + return null; // No data + } + + frameParser.reset(); + int consumed = frameParser.parse(recvBufPtr + recvReadPos, recvBufPtr + recvPos); + + if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE || + frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) { + return null; // Need more data + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) { + throw new HttpClientException("WebSocket frame parse error: ") + .put(WebSocketCloseCode.describe(frameParser.getErrorCode())); + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { + long payloadPtr = recvBufPtr + recvReadPos + frameParser.getHeaderSize(); + int payloadLen = (int) frameParser.getPayloadLength(); + + // Unmask if needed (server frames should not be masked) + if (frameParser.isMasked()) { + frameParser.unmaskPayload(payloadPtr, payloadLen); + } + + // Handle frame by opcode + int opcode = frameParser.getOpcode(); + switch (opcode) { + case WebSocketOpcode.PING: + // Auto-respond with pong + sendPongFrame(payloadPtr, payloadLen); + if (handler != null) { + handler.onPing(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.PONG: + if (handler != null) { + handler.onPong(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.CLOSE: + upgraded = false; + if (handler != null) { + int closeCode = 0; + String reason = null; + if (payloadLen >= 2) { + closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8) + | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF); + if (payloadLen > 2) { + byte[] reasonBytes = new byte[payloadLen - 2]; + for (int i = 0; i < reasonBytes.length; i++) { + reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i); + } + reason = new String(reasonBytes, StandardCharsets.UTF_8); + } + } + handler.onClose(closeCode, reason); + } + break; + case WebSocketOpcode.BINARY: + if (handler != null) { + handler.onBinaryMessage(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.TEXT: + if (handler != null) { + handler.onTextMessage(payloadPtr, payloadLen); + } + break; + } + + // Advance read position + recvReadPos += consumed; + + // Compact buffer if needed + compactRecvBuffer(); + + return true; + } + + return false; + } + + private void sendPongFrame(long payloadPtr, int payloadLen) { + try { + sendBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePongFrame(payloadPtr, payloadLen); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, 1000); // Short timeout for pong + sendBuffer.reset(); + } catch (Exception e) { + LOG.error("Failed to send pong: {}", e.getMessage()); + } + } + + private void compactRecvBuffer() { + if (recvReadPos > 0) { + int remaining = recvPos - recvReadPos; + if (remaining > 0) { + Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining); + } + recvPos = remaining; + recvReadPos = 0; + } + } + + private void growRecvBuffer() { + int newSize = recvBufSize * 2; + recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT); + recvBufSize = newSize; + } + + // === Socket I/O helpers === + + private int recvOrDie(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + int n = dieIfNegative(socket.recv(ptr, len)); + if (n == 0) { + ioWait(remainingTime(timeout, startTime), IOOperation.READ); + n = dieIfNegative(socket.recv(ptr, len)); + } + return n; + } + + private int recvOrTimeout(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + int n = socket.recv(ptr, len); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + if (n == 0) { + try { + ioWait(timeout, IOOperation.READ); + } catch (HttpClientException e) { + // Timeout + return 0; + } + n = socket.recv(ptr, len); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + } + return n; + } + + private int dieIfNegative(int byteCount) { + if (byteCount < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + return byteCount; + } + + private int remainingTime(int timeoutMillis, long startTimeNanos) { + timeoutMillis -= (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); + if (timeoutMillis <= 0) { + throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']'); + } + return timeoutMillis; + } + + protected void dieWaiting(int n) { + if (n == 1) { + return; + } + if (n == 0) { + throw new HttpClientException("timed out [errno=").put(nf.errno()).put(']'); + } + throw new HttpClientException("queue error [errno=").put(nf.errno()).put(']'); + } + + private void checkConnected() { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + if (!upgraded) { + throw new HttpClientException("WebSocket not connected or upgraded"); + } + } + + // === State === + + /** + * Returns whether the WebSocket is connected and upgraded. + */ + public boolean isConnected() { + return upgraded && !closed && !socket.isClosed(); + } + + /** + * Returns the connected host. + */ + public CharSequence getHost() { + return host; + } + + /** + * Returns the connected port. + */ + public int getPort() { + return port; + } + + // === Platform-specific I/O === + + /** + * Waits for I/O readiness using platform-specific mechanism. + * + * @param timeout timeout in milliseconds + * @param op I/O operation (READ or WRITE) + */ + protected abstract void ioWait(int timeout, int op); + + /** + * Sets up platform-specific I/O wait mechanism after connection. + */ + protected abstract void setupIoWait(); +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java new file mode 100644 index 0000000..9284786 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java @@ -0,0 +1,137 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.ClientTlsConfiguration; +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.JavaTlsClientSocketFactory; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Os; + +/** + * Factory for creating platform-specific {@link WebSocketClient} instances. + *

+ * Usage: + * 

+ * // Plain text connection
+ * WebSocketClient client = WebSocketClientFactory.newPlainTextInstance();
+ *
+ * // TLS connection
+ * WebSocketClient client = WebSocketClientFactory.newTlsInstance(config, tlsConfig);
+ *
+ * // Connect and upgrade
+ * client.connect("localhost", 9000);
+ * client.upgrade("/ws");
+ *
+ * // Send data
+ * WebSocketSendBuffer buf = client.getSendBuffer();
+ * buf.beginBinaryFrame();
+ * buf.putLong(data);
+ * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+ * client.sendFrame(frame);
+ * buf.reset();
+ *
+ * // Receive data
+ * client.receiveFrame(handler);
+ *
+ * client.close();
+ *
+ */ +public class WebSocketClientFactory { + + /** + * Creates a new WebSocket client with insecure TLS (no certificate validation). + *

+ * WARNING: Only use this for testing. Production code should use proper TLS validation. + * + * @return a new WebSocket client with insecure TLS + */ + public static WebSocketClient newInsecureTlsInstance() { + return newInstance(DefaultHttpClientConfiguration.INSTANCE, JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION); + } + + /** + * Creates a new WebSocket client with the specified configuration and socket factory. + * + * @param configuration the HTTP client configuration + * @param socketFactory the socket factory for creating sockets + * @return a new platform-specific WebSocket client + */ + public static WebSocketClient newInstance(HttpClientConfiguration configuration, SocketFactory socketFactory) { + switch (Os.type) { + case Os.LINUX: + return new WebSocketClientLinux(configuration, socketFactory); + case Os.DARWIN: + case Os.FREEBSD: + return new WebSocketClientOsx(configuration, socketFactory); + case Os.WINDOWS: + return new WebSocketClientWindows(configuration, socketFactory); + default: + throw new UnsupportedOperationException("Unsupported platform: " + Os.type); + } + } + + /** + * Creates a new plain text WebSocket client with default configuration. + * + * @return a new plain text WebSocket client + */ + public static WebSocketClient newPlainTextInstance() { + return newPlainTextInstance(DefaultHttpClientConfiguration.INSTANCE); + } + + /** + * Creates a new plain text WebSocket client with the specified configuration. + * + * @param configuration the HTTP client configuration + * @return a new plain text WebSocket client + */ + public static WebSocketClient newPlainTextInstance(HttpClientConfiguration configuration) { + return newInstance(configuration, PlainSocketFactory.INSTANCE); + } + + /** + * Creates a new TLS WebSocket client with the specified configuration. + * + * @param configuration the HTTP client configuration + * @param tlsConfig the TLS configuration + * @return a new TLS WebSocket client + */ + public static WebSocketClient newTlsInstance(HttpClientConfiguration configuration, ClientTlsConfiguration tlsConfig) { + return newInstance(configuration, new JavaTlsClientSocketFactory(tlsConfig)); + } + + /** + * Creates a new TLS WebSocket client with default HTTP configuration. + * + * @param tlsConfig the TLS configuration + * @return a new TLS WebSocket client + */ + public static WebSocketClient newTlsInstance(ClientTlsConfiguration tlsConfig) { + return newTlsInstance(DefaultHttpClientConfiguration.INSTANCE, tlsConfig); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java new file mode 100644 index 0000000..f4ac6ba --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.Epoll; +import io.questdb.client.network.EpollAccessor; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * Linux-specific WebSocket client using epoll for I/O waiting. + */ +public class WebSocketClientLinux extends WebSocketClient { + private Epoll epoll; + + public WebSocketClientLinux(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + epoll = new Epoll( + configuration.getEpollFacade(), + configuration.getWaitQueueCapacity() + ); + } + + @Override + public void close() { + super.close(); + epoll = Misc.free(epoll); + } + + @Override + protected void ioWait(int timeout, int op) { + final int event = op == IOOperation.WRITE ? EpollAccessor.EPOLLOUT : EpollAccessor.EPOLLIN; + if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_MOD, event) < 0) { + throw new HttpClientException("internal error: epoll_ctl failure [op=").put(op) + .put(", errno=").put(nf.errno()) + .put(']'); + } + dieWaiting(epoll.poll(timeout)); + } + + @Override + protected void setupIoWait() { + if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_ADD, EpollAccessor.EPOLLOUT) < 0) { + throw new HttpClientException("internal error: epoll_ctl failure [cmd=add, errno=").put(nf.errno()).put(']'); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java new file mode 100644 index 0000000..d34df7c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.Kqueue; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * macOS-specific WebSocket client using kqueue for I/O waiting. + */ +public class WebSocketClientOsx extends WebSocketClient { + private Kqueue kqueue; + + public WebSocketClientOsx(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + this.kqueue = new Kqueue( + configuration.getKQueueFacade(), + configuration.getWaitQueueCapacity() + ); + } + + @Override + public void close() { + super.close(); + this.kqueue = Misc.free(kqueue); + } + + @Override + protected void ioWait(int timeout, int op) { + kqueue.setWriteOffset(0); + if (op == IOOperation.READ) { + kqueue.readFD(socket.getFd(), 0); + } else { + kqueue.writeFD(socket.getFd(), 0); + } + + // 1 = always one FD, we are a single threaded network client + if (kqueue.register(1) != 0) { + throw new HttpClientException("could not register with kqueue [op=").put(op) + .put(", errno=").errno(nf.errno()) + .put(']'); + } + dieWaiting(kqueue.poll(timeout)); + } + + @Override + protected void setupIoWait() { + // no-op on macOS + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java new file mode 100644 index 0000000..cdaec88 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java @@ -0,0 +1,74 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.FDSet; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.SelectFacade; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * Windows-specific WebSocket client using select for I/O waiting. + */ +public class WebSocketClientWindows extends WebSocketClient { + private final SelectFacade sf; + private FDSet fdSet; + + public WebSocketClientWindows(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + this.fdSet = new FDSet(configuration.getWaitQueueCapacity()); + this.sf = configuration.getSelectFacade(); + } + + @Override + public void close() { + super.close(); + this.fdSet = Misc.free(fdSet); + } + + @Override + protected void ioWait(int timeout, int op) { + final long readAddr; + final long writeAddr; + fdSet.clear(); + fdSet.add(socket.getFd()); + fdSet.setCount(1); + if (op == IOOperation.READ) { + readAddr = fdSet.address(); + writeAddr = 0; + } else { + readAddr = 0; + writeAddr = fdSet.address(); + } + dieWaiting(sf.select(readAddr, writeAddr, 0, timeout)); + } + + @Override + protected void setupIoWait() { + // no-op on Windows + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java new file mode 100644 index 0000000..aff429d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java @@ -0,0 +1,93 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +/** + * Callback interface for handling received WebSocket frames. + *

+ * Implementations should process received data efficiently and avoid blocking, + * as callbacks are invoked on the I/O thread. + *

+ * Thread safety: Callbacks are invoked from the thread that called receiveFrame(). + * Implementations must handle their own synchronization if accessed from multiple threads. + */ +public interface WebSocketFrameHandler { + + /** + * Called when a binary frame is received. + * + * @param payloadPtr pointer to the payload data in native memory + * @param payloadLen length of the payload in bytes + */ + void onBinaryMessage(long payloadPtr, int payloadLen); + + /** + * Called when a text frame is received. + *

+ * Default implementation does nothing. Override if text frames need handling. + * + * @param payloadPtr pointer to the UTF-8 encoded payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onTextMessage(long payloadPtr, int payloadLen) { + // Default: ignore text frames + } + + /** + * Called when a close frame is received from the server. + *

+ * After this callback, the connection will be closed. The handler should + * perform any necessary cleanup. + * + * @param code the close status code (e.g., 1000 for normal closure) + * @param reason the close reason (may be null or empty) + */ + void onClose(int code, String reason); + + /** + * Called when a ping frame is received. + *

+ * Default implementation does nothing. The WebSocketClient automatically + * sends a pong response, so this callback is for informational purposes only. + * + * @param payloadPtr pointer to the ping payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onPing(long payloadPtr, int payloadLen) { + // Default: handled automatically by client + } + + /** + * Called when a pong frame is received. + *

+ * Default implementation does nothing. + * + * @param payloadPtr pointer to the pong payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onPong(long payloadPtr, int payloadLen) { + // Default: ignore pong frames + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java new file mode 100644 index 0000000..0eea869 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -0,0 +1,582 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.ilpv4.client.IlpBufferWriter; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; + +/** + * Zero-GC WebSocket send buffer that implements {@link ArrayBufferAppender} for direct + * payload writing. Manages native memory with safe growth and handles WebSocket frame + * building (reserve header -> write payload -> patch header -> mask). + *

+ * Usage pattern: + * 

+ * buffer.beginBinaryFrame();
+ * // Write payload using ArrayBufferAppender methods
+ * buffer.putLong(value);
+ * buffer.putBlockOfBytes(ptr, len);
+ * // Finish frame and get send info
+ * FrameInfo frame = buffer.endBinaryFrame();
+ * // Send frame using socket
+ * socket.send(buffer.getBufferPtr() + frame.offset, frame.length);
+ * buffer.reset();
+ *
+ *

+ * Thread safety: This class is NOT thread-safe. Each connection should have its own buffer. + */ +public class WebSocketSendBuffer implements IlpBufferWriter, QuietCloseable { + + // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key) + private static final int MAX_HEADER_SIZE = 14; + + private static final int DEFAULT_INITIAL_CAPACITY = 65536; + private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; // Leave room for alignment + + private long bufPtr; + private int bufCapacity; + private int writePos; // Current write position (offset from bufPtr) + private int frameStartOffset; // Where current frame's reserved header starts + private int payloadStartOffset; // Where payload begins (frameStart + MAX_HEADER_SIZE) + + private final Rnd rnd; + private final int maxBufferSize; + private final FrameInfo frameInfo = new FrameInfo(); + + /** + * Creates a new WebSocket send buffer with default initial capacity. + */ + public WebSocketSendBuffer() { + this(DEFAULT_INITIAL_CAPACITY, MAX_BUFFER_SIZE); + } + + /** + * Creates a new WebSocket send buffer with specified initial capacity. + * + * @param initialCapacity initial buffer size in bytes + */ + public WebSocketSendBuffer(int initialCapacity) { + this(initialCapacity, MAX_BUFFER_SIZE); + } + + /** + * Creates a new WebSocket send buffer with specified initial and max capacity. + * + * @param initialCapacity initial buffer size in bytes + * @param maxBufferSize maximum buffer size in bytes + */ + public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) { + this.bufCapacity = Math.max(initialCapacity, MAX_HEADER_SIZE * 2); + this.maxBufferSize = maxBufferSize; + this.bufPtr = Unsafe.malloc(bufCapacity, MemoryTag.NATIVE_DEFAULT); + this.writePos = 0; + this.frameStartOffset = 0; + this.payloadStartOffset = 0; + this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + } + + @Override + public void close() { + if (bufPtr != 0) { + Unsafe.free(bufPtr, bufCapacity, MemoryTag.NATIVE_DEFAULT); + bufPtr = 0; + bufCapacity = 0; + } + } + + // === Buffer Management === + + /** + * Ensures the buffer has capacity for the specified number of additional bytes. + * May reallocate the buffer if necessary. + * + * @param additionalBytes number of additional bytes needed + */ + @Override + public void ensureCapacity(int additionalBytes) { + long requiredCapacity = (long) writePos + additionalBytes; + if (requiredCapacity > bufCapacity) { + grow(requiredCapacity); + } + } + + private void grow(long requiredCapacity) { + if (requiredCapacity > maxBufferSize) { + throw new HttpClientException("WebSocket buffer size exceeded maximum [required=") + .put(requiredCapacity) + .put(", max=") + .put(maxBufferSize) + .put(']'); + } + int newCapacity = (int) Math.min( + Numbers.ceilPow2((int) requiredCapacity), + maxBufferSize + ); + bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT); + bufCapacity = newCapacity; + } + + // === ArrayBufferAppender Implementation === + + @Override + public void putByte(byte b) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(bufPtr + writePos, b); + writePos++; + } + + @Override + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(bufPtr + writePos, value); + writePos += 4; + } + + @Override + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufPtr + writePos, value); + writePos += 8; + } + + @Override + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(bufPtr + writePos, value); + writePos += 8; + } + + @Override + public void putBlockOfBytes(long from, long len) { + if (len <= 0) { + return; + } + ensureCapacity((int) len); + Vect.memcpy(bufPtr + writePos, from, len); + writePos += (int) len; + } + + // === Additional write methods (not in ArrayBufferAppender but useful) === + + /** + * Writes a short value in little-endian format. + */ + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(bufPtr + writePos, value); + writePos += 2; + } + + /** + * Writes a float value. + */ + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(bufPtr + writePos, value); + writePos += 4; + } + + /** + * Writes a long value in big-endian format. + */ + public void putLongBE(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufPtr + writePos, Long.reverseBytes(value)); + writePos += 8; + } + + /** + * Writes raw bytes from a byte array. + */ + public void putBytes(byte[] bytes, int offset, int length) { + if (length <= 0) { + return; + } + ensureCapacity(length); + for (int i = 0; i < length; i++) { + Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]); + } + writePos += length; + } + + /** + * Writes an ASCII string. + */ + public void putAscii(CharSequence cs) { + if (cs == null) { + return; + } + int len = cs.length(); + ensureCapacity(len); + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i)); + } + writePos += len; + } + + // === IlpBufferWriter Implementation === + + /** + * Writes an unsigned variable-length integer (LEB128 encoding). + */ + @Override + public void putVarint(long value) { + while (value > 0x7F) { + putByte((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + putByte((byte) value); + } + + /** + * Writes a length-prefixed UTF-8 string. + */ + @Override + public void putString(String value) { + if (value == null || value.isEmpty()) { + putVarint(0); + return; + } + int utf8Len = IlpBufferWriter.utf8Length(value); + putVarint(utf8Len); + putUtf8(value); + } + + /** + * Writes UTF-8 encoded bytes directly without length prefix. + */ + @Override + public void putUtf8(String value) { + if (value == null || value.isEmpty()) { + return; + } + for (int i = 0, n = value.length(); i < n; i++) { + char c = value.charAt(i); + if (c < 0x80) { + putByte((byte) c); + } else if (c < 0x800) { + putByte((byte) (0xC0 | (c >> 6))); + putByte((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + char c2 = value.charAt(++i); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) (0xE0 | (c >> 12))); + putByte((byte) (0x80 | ((c >> 6) & 0x3F))); + putByte((byte) (0x80 | (c & 0x3F))); + } + } + } + + /** + * Patches an int value at the specified offset. + */ + @Override + public void patchInt(int offset, int value) { + Unsafe.getUnsafe().putInt(bufPtr + offset, value); + } + + /** + * Skips the specified number of bytes, advancing the position. + */ + @Override + public void skip(int bytes) { + ensureCapacity(bytes); + writePos += bytes; + } + + /** + * Gets the current write position (number of bytes written). + */ + @Override + public int getPosition() { + return writePos; + } + + // === Frame Building === + + /** + * Begins a new binary WebSocket frame. Reserves space for the maximum header size. + * After calling this method, use ArrayBufferAppender methods to write the payload. + */ + public void beginBinaryFrame() { + beginFrame(WebSocketOpcode.BINARY); + } + + /** + * Begins a new text WebSocket frame. Reserves space for the maximum header size. + */ + public void beginTextFrame() { + beginFrame(WebSocketOpcode.TEXT); + } + + /** + * Begins a new WebSocket frame with the specified opcode. + * + * @param opcode the frame opcode + */ + public void beginFrame(int opcode) { + frameStartOffset = writePos; + // Reserve maximum header space + ensureCapacity(MAX_HEADER_SIZE); + writePos += MAX_HEADER_SIZE; + payloadStartOffset = writePos; + } + + /** + * Finishes the current binary frame, writing the header and applying masking. + * Returns information about where to find the complete frame in the buffer. + *

+ * IMPORTANT: Only call this after all payload writes are complete. The buffer + * pointer is stable after this call (no more reallocations for this frame). + * + * @return frame info containing offset and length for sending + */ + public FrameInfo endBinaryFrame() { + return endFrame(WebSocketOpcode.BINARY); + } + + /** + * Finishes the current text frame, writing the header and applying masking. + */ + public FrameInfo endTextFrame() { + return endFrame(WebSocketOpcode.TEXT); + } + + /** + * Finishes the current frame with the specified opcode. + * + * @param opcode the frame opcode + * @return frame info containing offset and length for sending + */ + public FrameInfo endFrame(int opcode) { + int payloadLen = writePos - payloadStartOffset; + + // Calculate actual header size (with mask key for client frames) + int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize; + int actualFrameStart = frameStartOffset + unusedSpace; + + // Generate mask key + int maskKey = rnd.nextInt(); + + // Write header at actual position (after unused space) + WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey); + + // Apply mask to payload + if (payloadLen > 0) { + WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey); + } + + return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen); + } + + /** + * Writes a complete ping frame (control frame, no masking needed for server). + * Note: Client frames MUST be masked per RFC 6455. This writes a masked ping. + * + * @return frame info for sending + */ + public FrameInfo writePingFrame() { + return writePingFrame(0, 0); + } + + /** + * Writes a complete ping frame with payload. + * + * @param payloadPtr pointer to ping payload + * @param payloadLen length of payload (max 125 bytes for control frames) + * @return frame info for sending + */ + public FrameInfo writePingFrame(long payloadPtr, int payloadLen) { + if (payloadLen > 125) { + throw new HttpClientException("Ping payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PING, payloadLen, maskKey); + writePos += written; + + if (payloadLen > 0) { + Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen); + WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey); + writePos += payloadLen; + } + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + /** + * Writes a complete pong frame. + * + * @param payloadPtr pointer to pong payload (should match received ping) + * @param payloadLen length of payload + * @return frame info for sending + */ + public FrameInfo writePongFrame(long payloadPtr, int payloadLen) { + if (payloadLen > 125) { + throw new HttpClientException("Pong payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PONG, payloadLen, maskKey); + writePos += written; + + if (payloadLen > 0) { + Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen); + WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey); + writePos += payloadLen; + } + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + /** + * Writes a complete close frame. + * + * @param code close status code (e.g., 1000 for normal closure) + * @param reason optional reason string (may be null) + * @return frame info for sending + */ + public FrameInfo writeCloseFrame(int code, String reason) { + int payloadLen = 2; // status code + byte[] reasonBytes = null; + if (reason != null && !reason.isEmpty()) { + reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8); + payloadLen += reasonBytes.length; + } + + if (payloadLen > 125) { + throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey); + writePos += written; + + // Write status code (big-endian) + long payloadStart = bufPtr + writePos; + Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF)); + writePos += 2; + + // Write reason if present + if (reasonBytes != null) { + for (byte reasonByte : reasonBytes) { + Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte); + } + } + + // Mask the payload (including status code and reason) + WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey); + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + // === Buffer State === + + /** + * Gets the buffer pointer. Only use this for reading after frame is complete. + */ + public long getBufferPtr() { + return bufPtr; + } + + /** + * Gets the current buffer capacity. + */ + public int getCapacity() { + return bufCapacity; + } + + /** + * Gets the current write position (total bytes written since last reset). + */ + public int getWritePos() { + return writePos; + } + + /** + * Gets the payload length of the current frame being built. + */ + public int getCurrentPayloadLength() { + return writePos - payloadStartOffset; + } + + /** + * Resets the buffer for reuse. Does not deallocate memory. + */ + public void reset() { + writePos = 0; + frameStartOffset = 0; + payloadStartOffset = 0; + } + + /** + * Information about a completed WebSocket frame's location in the buffer. + * This class is mutable and reused to avoid allocations. Callers must + * extract values before calling any end*Frame() method again. + */ + public static final class FrameInfo { + /** + * Offset from buffer start where the frame begins. + */ + public int offset; + + /** + * Total length of the frame (header + payload). + */ + public int length; + + FrameInfo set(int offset, int length) { + this.offset = offset; + this.length = length; + return this; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java new file mode 100644 index 0000000..743c029 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java @@ -0,0 +1,173 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.CharSequenceIntHashMap; +import io.questdb.client.std.ObjList; + +/** + * Global symbol dictionary that maps symbol strings to sequential integer IDs. + *

+ * This dictionary is shared across all tables and columns within a client instance. + * IDs are assigned sequentially starting from 0, ensuring contiguous ID space. + *

+ * Thread safety: This class is NOT thread-safe. External synchronization is required + * if accessed from multiple threads. + */ +public class GlobalSymbolDictionary { + + private final CharSequenceIntHashMap symbolToId; + private final ObjList idToSymbol; + + public GlobalSymbolDictionary() { + this(64); // Default initial capacity + } + + public GlobalSymbolDictionary(int initialCapacity) { + this.symbolToId = new CharSequenceIntHashMap(initialCapacity); + this.idToSymbol = new ObjList<>(initialCapacity); + } + + /** + * Gets or adds a symbol to the dictionary. + *

+ * If the symbol already exists, returns its existing ID. + * If the symbol is new, assigns the next sequential ID and returns it. + * + * @param symbol the symbol string (must not be null) + * @return the global ID for this symbol (>= 0) + * @throws IllegalArgumentException if symbol is null + */ + public int getOrAddSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + + int existingId = symbolToId.get(symbol); + if (existingId != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + return existingId; + } + + // Assign new ID + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + + /** + * Gets the symbol string for a given ID. + * + * @param id the symbol ID + * @return the symbol string + * @throws IndexOutOfBoundsException if id is out of range + */ + public String getSymbol(int id) { + if (id < 0 || id >= idToSymbol.size()) { + throw new IndexOutOfBoundsException("Invalid symbol ID: " + id + ", dictionary size: " + idToSymbol.size()); + } + return idToSymbol.getQuick(id); + } + + /** + * Gets the ID for an existing symbol, or -1 if not found. + * + * @param symbol the symbol string + * @return the symbol ID, or -1 if not in dictionary + */ + public int getId(String symbol) { + if (symbol == null) { + return -1; + } + int id = symbolToId.get(symbol); + return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id; + } + + /** + * Returns the number of symbols in the dictionary. + * + * @return dictionary size + */ + public int size() { + return idToSymbol.size(); + } + + /** + * Checks if the dictionary is empty. + * + * @return true if no symbols have been added + */ + public boolean isEmpty() { + return idToSymbol.size() == 0; + } + + /** + * Checks if the dictionary contains the given symbol. + * + * @param symbol the symbol to check + * @return true if the symbol exists in the dictionary + */ + public boolean contains(String symbol) { + return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + } + + /** + * Gets the symbols in the given ID range [fromId, toId). + *

+ * This is used to extract the delta for sending to the server. + * The range is inclusive of fromId and exclusive of toId. + * + * @param fromId start ID (inclusive) + * @param toId end ID (exclusive) + * @return array of symbols in the range, or empty array if range is invalid/empty + */ + public String[] getSymbolsInRange(int fromId, int toId) { + if (fromId < 0 || toId < fromId || fromId >= idToSymbol.size()) { + return new String[0]; + } + + int actualToId = Math.min(toId, idToSymbol.size()); + int count = actualToId - fromId; + if (count <= 0) { + return new String[0]; + } + + String[] result = new String[count]; + for (int i = 0; i < count; i++) { + result[i] = idToSymbol.getQuick(fromId + i); + } + return result; + } + + /** + * Clears all symbols from the dictionary. + *

+ * After clearing, the next symbol added will get ID 0. + */ + public void clear() { + symbolToId.clear(); + idToSymbol.clear(); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java new file mode 100644 index 0000000..1976cac --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java @@ -0,0 +1,177 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; + +/** + * Buffer writer interface for ILP v4 message encoding. + *

+ * This interface extends {@link ArrayBufferAppender} with additional methods + * required for encoding ILP v4 messages, including varint encoding, string + * handling, and buffer manipulation. + *

+ * Implementations include: + *

+ *

+ * All multi-byte values are written in little-endian format unless the method + * name explicitly indicates big-endian (e.g., {@link #putLongBE}). + */ +public interface IlpBufferWriter extends ArrayBufferAppender { + + // === Primitive writes (little-endian) === + + /** + * Writes a short (2 bytes, little-endian). + */ + void putShort(short value); + + /** + * Writes a float (4 bytes, little-endian). + */ + void putFloat(float value); + + // === Big-endian writes === + + /** + * Writes a long in big-endian byte order. + */ + void putLongBE(long value); + + // === Variable-length encoding === + + /** + * Writes an unsigned variable-length integer (LEB128 encoding). + *

+ * Each byte contains 7 bits of data with the high bit indicating + * whether more bytes follow. + */ + void putVarint(long value); + + // === String encoding === + + /** + * Writes a length-prefixed UTF-8 string. + *

+ * Format: varint length + UTF-8 bytes + * + * @param value the string to write (may be null or empty) + */ + void putString(String value); + + /** + * Writes UTF-8 encoded bytes directly without length prefix. + * + * @param value the string to encode (may be null or empty) + */ + void putUtf8(String value); + + // === Buffer manipulation === + + /** + * Patches an int value at the specified offset in the buffer. + *

+ * Used for updating length fields after writing content. + * + * @param offset the byte offset from buffer start + * @param value the int value to write + */ + void patchInt(int offset, int value); + + /** + * Skips the specified number of bytes, advancing the position. + *

+ * Used when data has been written directly to the buffer via + * {@link #getBufferPtr()}. + * + * @param bytes number of bytes to skip + */ + void skip(int bytes); + + /** + * Ensures the buffer has capacity for at least the specified + * additional bytes beyond the current position. + * + * @param additionalBytes number of additional bytes needed + */ + void ensureCapacity(int additionalBytes); + + /** + * Resets the buffer for reuse, setting the position to 0. + *

+ * Does not deallocate memory. + */ + void reset(); + + // === Buffer state === + + /** + * Returns the current write position (number of bytes written). + */ + int getPosition(); + + /** + * Returns the current buffer capacity in bytes. + */ + int getCapacity(); + + /** + * Returns the native memory pointer to the buffer start. + *

+ * The returned pointer is valid until the next buffer growth operation. + * Use with care and only for reading completed data. + */ + long getBufferPtr(); + + // === Utility === + + /** + * Returns the UTF-8 encoded length of a string. + * + * @param s the string (may be null) + * @return the number of bytes needed to encode the string as UTF-8 + */ + static int utf8Length(String s) { + if (s == null) return 0; + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + i++; + len += 4; + } else { + len += 3; + } + } + return len; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java new file mode 100644 index 0000000..f1826f5 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java @@ -0,0 +1,790 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.ilpv4.protocol.*; + +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4ColumnDef; +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4GorillaEncoder; + +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TimestampDecoder; +import io.questdb.client.std.QuietCloseable; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * Encodes ILP v4 messages for WebSocket transport. + *

+ * This encoder can write to either an internal {@link NativeBufferWriter} (default) + * or an external {@link IlpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}. + *

+ * When using an external buffer, the encoder writes directly to it without intermediate copies, + * enabling zero-copy WebSocket frame construction. + *

+ * Usage with external buffer (zero-copy): + * 

+ * WebSocketSendBuffer buf = client.getSendBuffer();
+ * buf.beginBinaryFrame();
+ * encoder.setBuffer(buf);
+ * encoder.encode(tableData, false);
+ * FrameInfo frame = buf.endBinaryFrame();
+ * client.sendFrame(frame);
+ *
+ */ +public class IlpV4WebSocketEncoder implements QuietCloseable { + + private NativeBufferWriter ownedBuffer; + private IlpBufferWriter buffer; + private final IlpV4GorillaEncoder gorillaEncoder = new IlpV4GorillaEncoder(); + private byte flags; + + public IlpV4WebSocketEncoder() { + this.ownedBuffer = new NativeBufferWriter(); + this.buffer = ownedBuffer; + this.flags = 0; + } + + public IlpV4WebSocketEncoder(int bufferSize) { + this.ownedBuffer = new NativeBufferWriter(bufferSize); + this.buffer = ownedBuffer; + this.flags = 0; + } + + /** + * Returns the underlying buffer. + *

+ * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, + * that buffer is returned. Otherwise, returns the internal buffer. + */ + public IlpBufferWriter getBuffer() { + return buffer; + } + + /** + * Sets an external buffer for encoding. + *

+ * When set, the encoder writes directly to this buffer instead of its internal buffer. + * The caller is responsible for managing the external buffer's lifecycle. + *

+ * Pass {@code null} to revert to using the internal buffer. + * + * @param externalBuffer the external buffer to use, or null to use internal buffer + */ + public void setBuffer(IlpBufferWriter externalBuffer) { + this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; + } + + /** + * Returns true if currently using an external buffer. + */ + public boolean isUsingExternalBuffer() { + return buffer != ownedBuffer; + } + + /** + * Resets the encoder for a new message. + *

+ * If using an external buffer, this only resets the internal state (flags). + * The external buffer's reset is the caller's responsibility. + * If using the internal buffer, resets both the buffer and internal state. + */ + public void reset() { + if (!isUsingExternalBuffer()) { + buffer.reset(); + } + } + + /** + * Sets whether Gorilla timestamp encoding is enabled. + */ + public void setGorillaEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_GORILLA; + } else { + flags &= ~FLAG_GORILLA; + } + } + + /** + * Returns true if Gorilla encoding is enabled. + */ + public boolean isGorillaEnabled() { + return (flags & FLAG_GORILLA) != 0; + } + + /** + * Encodes a complete ILP v4 message from a table buffer. + * + * @param tableBuffer the table buffer containing row data + * @param useSchemaRef whether to use schema reference mode + * @return the number of bytes written + */ + public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + buffer.reset(); + + // Write message header with placeholder for payload length + writeHeader(1, 0); + int payloadStart = buffer.getPosition(); + + // Encode table data + encodeTable(tableBuffer, useSchemaRef); + + // Patch payload length + int payloadLength = buffer.getPosition() - payloadStart; + buffer.patchInt(8, payloadLength); + + return buffer.getPosition(); + } + + /** + * Encodes a complete ILP v4 message with delta symbol dictionary encoding. + *

+ * This method sends only new symbols (delta) since the last confirmed watermark, + * and uses global symbol IDs instead of per-column local indices. + * + * @param tableBuffer the table buffer containing row data + * @param globalDict the global symbol dictionary + * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) + * @param batchMaxId the highest symbol ID used in this batch + * @param useSchemaRef whether to use schema reference mode + * @return the number of bytes written + */ + public int encodeWithDeltaDict( + IlpV4TableBuffer tableBuffer, + GlobalSymbolDictionary globalDict, + int confirmedMaxId, + int batchMaxId, + boolean useSchemaRef + ) { + buffer.reset(); + + // Calculate delta range + int deltaStart = confirmedMaxId + 1; + int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + + // Set delta dictionary flag + byte savedFlags = flags; + flags |= FLAG_DELTA_SYMBOL_DICT; + + // Write message header with placeholder for payload length + writeHeader(1, 0); + int payloadStart = buffer.getPosition(); + + // Write symbol delta section (before tables) + buffer.putVarint(deltaStart); + buffer.putVarint(deltaCount); + for (int id = deltaStart; id < deltaStart + deltaCount; id++) { + String symbol = globalDict.getSymbol(id); + buffer.putString(symbol); + } + + // Encode table data (symbol columns will use global IDs) + encodeTableWithGlobalSymbols(tableBuffer, useSchemaRef); + + // Patch payload length + int payloadLength = buffer.getPosition() - payloadStart; + buffer.patchInt(8, payloadLength); + + // Restore flags + flags = savedFlags; + + return buffer.getPosition(); + } + + /** + * Sets the delta symbol dictionary flag. + */ + public void setDeltaSymbolDictEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_DELTA_SYMBOL_DICT; + } else { + flags &= ~FLAG_DELTA_SYMBOL_DICT; + } + } + + /** + * Returns true if delta symbol dictionary encoding is enabled. + */ + public boolean isDeltaSymbolDictEnabled() { + return (flags & FLAG_DELTA_SYMBOL_DICT) != 0; + } + + /** + * Writes the ILP v4 message header. + * + * @param tableCount number of tables in the message + * @param payloadLength payload length (can be 0 if patched later) + */ + public void writeHeader(int tableCount, int payloadLength) { + // Magic "ILP4" + buffer.putByte((byte) 'I'); + buffer.putByte((byte) 'L'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '4'); + + // Version + buffer.putByte(VERSION_1); + + // Flags + buffer.putByte(flags); + + // Table count (uint16, little-endian) + buffer.putShort((short) tableCount); + + // Payload length (uint32, little-endian) + buffer.putInt(payloadLength); + } + + /** + * Encodes a single table from the buffer. + */ + private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumn(col, colDef, rowCount, useGorilla); + } + } + + /** + * Encodes a single table from the buffer using global symbol IDs. + * This is used with delta dictionary encoding. + */ + private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + } + } + + /** + * Writes a table header with full schema. + */ + private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { + // Table name + buffer.putString(tableName); + + // Row count (varint) + buffer.putVarint(rowCount); + + // Column count (varint) + buffer.putVarint(columns.length); + + // Schema mode: full schema (0x00) + buffer.putByte(SCHEMA_MODE_FULL); + + // Column definitions (name + type for each) + for (IlpV4ColumnDef col : columns) { + buffer.putString(col.getName()); + buffer.putByte(col.getWireTypeCode()); + } + } + + /** + * Writes a table header with schema reference. + */ + private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { + // Table name + buffer.putString(tableName); + + // Row count (varint) + buffer.putVarint(rowCount); + + // Column count (varint) + buffer.putVarint(columnCount); + + // Schema mode: reference (0x01) + buffer.putByte(SCHEMA_MODE_REFERENCE); + + // Schema hash (8 bytes) + buffer.putLong(schemaHash); + } + + /** + * Encodes a single column. + */ + private void encodeColumn(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + int valueCount = col.getValueCount(); + + // Write null bitmap if column is nullable + if (colDef.isNullable()) { + writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + } + + // Write column data based on type + switch (col.getType()) { + case TYPE_BOOLEAN: + writeBooleanColumn(col.getBooleanValues(), valueCount); + break; + case TYPE_BYTE: + writeByteColumn(col.getByteValues(), valueCount); + break; + case TYPE_SHORT: + case TYPE_CHAR: + writeShortColumn(col.getShortValues(), valueCount); + break; + case TYPE_INT: + writeIntColumn(col.getIntValues(), valueCount); + break; + case TYPE_LONG: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_FLOAT: + writeFloatColumn(col.getFloatValues(), valueCount); + break; + case TYPE_DOUBLE: + writeDoubleColumn(col.getDoubleValues(), valueCount); + break; + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + break; + case TYPE_DATE: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + writeStringColumn(col.getStringValues(), valueCount); + break; + case TYPE_SYMBOL: + writeSymbolColumn(col, valueCount); + break; + case TYPE_UUID: + writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + break; + case TYPE_LONG256: + writeLong256Column(col.getLong256Values(), valueCount); + break; + case TYPE_DOUBLE_ARRAY: + writeDoubleArrayColumn(col, valueCount); + break; + case TYPE_LONG_ARRAY: + writeLongArrayColumn(col, valueCount); + break; + case TYPE_DECIMAL64: + writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + break; + case TYPE_DECIMAL128: + writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + break; + case TYPE_DECIMAL256: + writeDecimal256Column(col.getDecimalScale(), + col.getDecimal256Hh(), col.getDecimal256Hl(), + col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + break; + default: + throw new IllegalStateException("Unknown column type: " + col.getType()); + } + } + + /** + * Encodes a single column using global symbol IDs for SYMBOL type. + * All other column types are encoded the same as encodeColumn. + */ + private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + int valueCount = col.getValueCount(); + + // Write null bitmap if column is nullable + if (colDef.isNullable()) { + writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + } + + // For symbol columns, use global IDs; for all others, use standard encoding + if (col.getType() == TYPE_SYMBOL) { + writeSymbolColumnWithGlobalIds(col, valueCount); + } else { + // Write column data based on type (same as encodeColumn) + switch (col.getType()) { + case TYPE_BOOLEAN: + writeBooleanColumn(col.getBooleanValues(), valueCount); + break; + case TYPE_BYTE: + writeByteColumn(col.getByteValues(), valueCount); + break; + case TYPE_SHORT: + case TYPE_CHAR: + writeShortColumn(col.getShortValues(), valueCount); + break; + case TYPE_INT: + writeIntColumn(col.getIntValues(), valueCount); + break; + case TYPE_LONG: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_FLOAT: + writeFloatColumn(col.getFloatValues(), valueCount); + break; + case TYPE_DOUBLE: + writeDoubleColumn(col.getDoubleValues(), valueCount); + break; + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + break; + case TYPE_DATE: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + writeStringColumn(col.getStringValues(), valueCount); + break; + case TYPE_UUID: + writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + break; + case TYPE_LONG256: + writeLong256Column(col.getLong256Values(), valueCount); + break; + case TYPE_DOUBLE_ARRAY: + writeDoubleArrayColumn(col, valueCount); + break; + case TYPE_LONG_ARRAY: + writeLongArrayColumn(col, valueCount); + break; + case TYPE_DECIMAL64: + writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + break; + case TYPE_DECIMAL128: + writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + break; + case TYPE_DECIMAL256: + writeDecimal256Column(col.getDecimalScale(), + col.getDecimal256Hh(), col.getDecimal256Hl(), + col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + break; + default: + throw new IllegalStateException("Unknown column type: " + col.getType()); + } + } + } + + /** + * Writes a null bitmap from bit-packed long array. + */ + private void writeNullBitmapPacked(long[] nullsPacked, int count) { + int bitmapSize = (count + 7) / 8; + + for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { + int longIndex = byteIdx >>> 3; + int byteInLong = byteIdx & 7; + byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); + buffer.putByte(b); + } + } + + /** + * Writes boolean column data (bit-packed). + */ + private void writeBooleanColumn(boolean[] values, int count) { + int packedSize = (count + 7) / 8; + + for (int i = 0; i < packedSize; i++) { + byte b = 0; + for (int bit = 0; bit < 8; bit++) { + int idx = i * 8 + bit; + if (idx < count && values[idx]) { + b |= (1 << bit); + } + } + buffer.putByte(b); + } + } + + private void writeByteColumn(byte[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putByte(values[i]); + } + } + + private void writeShortColumn(short[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putShort(values[i]); + } + } + + private void writeIntColumn(int[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putInt(values[i]); + } + } + + private void writeLongColumn(long[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putLong(values[i]); + } + } + + private void writeFloatColumn(float[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putFloat(values[i]); + } + } + + private void writeDoubleColumn(double[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putDouble(values[i]); + } + } + + /** + * Writes a timestamp column with optional Gorilla compression. + *

+ * When Gorilla encoding is enabled and applicable (3+ timestamps with + * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. + * Otherwise, falls back to uncompressed encoding. + */ + private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { + if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { + // Write Gorilla encoding flag + buffer.putByte(IlpV4TimestampDecoder.ENCODING_GORILLA); + + // Calculate size needed and ensure buffer has capacity + int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); + buffer.ensureCapacity(encodedSize); + + // Encode timestamps to buffer + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + values, + count + ); + buffer.skip(bytesWritten); + } else { + // Write uncompressed + if (useGorilla) { + buffer.putByte(IlpV4TimestampDecoder.ENCODING_UNCOMPRESSED); + } + writeLongColumn(values, count); + } + } + + /** + * Writes a string column with offset array. + */ + private void writeStringColumn(String[] strings, int count) { + // Calculate total data length + int totalDataLen = 0; + for (int i = 0; i < count; i++) { + if (strings[i] != null) { + totalDataLen += IlpBufferWriter.utf8Length(strings[i]); + } + } + + // Write offset array + int runningOffset = 0; + buffer.putInt(0); + for (int i = 0; i < count; i++) { + if (strings[i] != null) { + runningOffset += IlpBufferWriter.utf8Length(strings[i]); + } + buffer.putInt(runningOffset); + } + + // Write string data + for (int i = 0; i < count; i++) { + if (strings[i] != null) { + buffer.putUtf8(strings[i]); + } + } + } + + /** + * Writes a symbol column with dictionary. + * Format: + * - Dictionary length (varint) + * - Dictionary entries (length-prefixed UTF-8 strings) + * - Symbol indices (varints, one per value) + */ + private void writeSymbolColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + // Get symbol data from column buffer + int[] symbolIndices = col.getSymbolIndices(); + String[] dictionary = col.getSymbolDictionary(); + + // Write dictionary + buffer.putVarint(dictionary.length); + for (String symbol : dictionary) { + buffer.putString(symbol); + } + + // Write symbol indices (one per non-null value) + for (int i = 0; i < count; i++) { + buffer.putVarint(symbolIndices[i]); + } + } + + /** + * Writes a symbol column using global IDs (for delta dictionary mode). + * Format: + * - Global symbol IDs (varints, one per value) + *

+ * The dictionary is not included here because it's written at the message level + * in delta format. + */ + private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, int count) { + int[] globalIds = col.getGlobalSymbolIds(); + if (globalIds == null) { + // Fall back to local indices if no global IDs stored + int[] symbolIndices = col.getSymbolIndices(); + for (int i = 0; i < count; i++) { + buffer.putVarint(symbolIndices[i]); + } + } else { + // Write global symbol IDs + for (int i = 0; i < count; i++) { + buffer.putVarint(globalIds[i]); + } + } + } + + private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { + // Little-endian: lo first, then hi + for (int i = 0; i < count; i++) { + buffer.putLong(lowBits[i]); + buffer.putLong(highBits[i]); + } + } + + private void writeLong256Column(long[] values, int count) { + // Flat array: 4 longs per value, little-endian (least significant first) + // values layout: [long0, long1, long2, long3] per row + for (int i = 0; i < count * 4; i++) { + buffer.putLong(values[i]); + } + } + + private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putDouble(data[dataIdx++]); + } + } + } + + private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putLong(data[dataIdx++]); + } + } + } + + private void writeDecimal64Column(byte scale, long[] values, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(values[i]); + } + } + + private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(high[i]); + buffer.putLongBE(low[i]); + } + } + + private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(hh[i]); + buffer.putLongBE(hl[i]); + buffer.putLongBE(lh[i]); + buffer.putLongBE(ll[i]); + } + } + + @Override + public void close() { + if (ownedBuffer != null) { + ownedBuffer.close(); + ownedBuffer = null; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java new file mode 100644 index 0000000..43a0b4c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java @@ -0,0 +1,1398 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.ilpv4.protocol.*; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.std.Chars; +import io.questdb.client.std.CharSequenceObjHashMap; +import io.questdb.client.std.LongHashSet; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.bytes.DirectByteSlice; +import org.jetbrains.annotations.NotNull; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * ILP v4 WebSocket client sender for streaming data to QuestDB. + *

+ * This sender uses a double-buffering scheme with asynchronous I/O for high throughput: + *

+ *

+ * Configuration options: + *

+ *

+ * Example usage: + * 

+ * try (IlpV4WebSocketSender sender = IlpV4WebSocketSender.connect("localhost", 9000)) {
+ *     for (int i = 0; i < 100_000; i++) {
+ *         sender.table("metrics")
+ *               .symbol("host", "server-" + (i % 10))
+ *               .doubleColumn("cpu", Math.random() * 100)
+ *               .atNow();
+ *         // Rows are batched and sent asynchronously!
+ *     }
+ *     // flush() waits for all pending batches to be sent
+ *     sender.flush();
+ * }
+ *
+ */ +public class IlpV4WebSocketSender implements Sender { + + private static final Logger LOG = LoggerFactory.getLogger(IlpV4WebSocketSender.class); + + private static final int DEFAULT_BUFFER_SIZE = 8192; + private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB + public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; + public static final int DEFAULT_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB + public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms + public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 + public static final int DEFAULT_SEND_QUEUE_CAPACITY = WebSocketSendQueue.DEFAULT_QUEUE_CAPACITY; // 16 + private static final String WRITE_PATH = "/write/v4"; + + private final String host; + private final int port; + private final boolean tlsEnabled; + private final CharSequenceObjHashMap tableBuffers; + private IlpV4TableBuffer currentTableBuffer; + private String currentTableName; + // Cached column references to avoid repeated hashmap lookups + private IlpV4TableBuffer.ColumnBuffer cachedTimestampColumn; + private IlpV4TableBuffer.ColumnBuffer cachedTimestampNanosColumn; + + // Encoder for ILP v4 messages + private final IlpV4WebSocketEncoder encoder; + + // WebSocket client (zero-GC native implementation) + private WebSocketClient client; + private boolean connected; + private boolean closed; + + // Double-buffering for async I/O + private MicrobatchBuffer buffer0; + private MicrobatchBuffer buffer1; + private MicrobatchBuffer activeBuffer; + private WebSocketSendQueue sendQueue; + + // Flow control + private InFlightWindow inFlightWindow; + + // Auto-flush configuration + private final int autoFlushRows; + private final int autoFlushBytes; + private final long autoFlushIntervalNanos; + + // Flow control configuration + private final int inFlightWindowSize; + private final int sendQueueCapacity; + + // Configuration + private boolean gorillaEnabled = true; + + // Async mode: pending row tracking + private int pendingRowCount; + private long firstPendingRowTimeNanos; + + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + + // Global symbol dictionary for delta encoding + private final GlobalSymbolDictionary globalSymbolDictionary; + + // Track max global symbol ID used in current batch (for delta calculation) + private int currentBatchMaxSymbolId = -1; + + // Track highest symbol ID sent to server (for delta encoding) + // Once sent over TCP, server is guaranteed to receive it (or connection dies) + private volatile int maxSentSymbolId = -1; + + // Track schema hashes that have been sent to the server (for schema reference mode) + // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. + // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. + private final LongHashSet sentSchemaHashes = new LongHashSet(); + + private IlpV4WebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, + int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, + int inFlightWindowSize, int sendQueueCapacity) { + this.host = host; + this.port = port; + this.tlsEnabled = tlsEnabled; + this.encoder = new IlpV4WebSocketEncoder(bufferSize); + this.tableBuffers = new CharSequenceObjHashMap<>(); + this.currentTableBuffer = null; + this.currentTableName = null; + this.connected = false; + this.closed = false; + this.autoFlushRows = autoFlushRows; + this.autoFlushBytes = autoFlushBytes; + this.autoFlushIntervalNanos = autoFlushIntervalNanos; + this.inFlightWindowSize = inFlightWindowSize; + this.sendQueueCapacity = sendQueueCapacity; + + // Initialize global symbol dictionary for delta encoding + this.globalSymbolDictionary = new GlobalSymbolDictionary(); + + // Initialize double-buffering if async mode (window > 1) + if (inFlightWindowSize > 1) { + int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); + this.buffer0 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + this.buffer1 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + this.activeBuffer = buffer0; + } + } + + /** + * Creates a new sender and connects to the specified host and port. + * Uses synchronous mode for backward compatibility. + * + * @param host server host + * @param port server HTTP port (WebSocket upgrade happens on same port) + * @return connected sender + */ + public static IlpV4WebSocketSender connect(String host, int port) { + return connect(host, port, false); + } + + /** + * Creates a new sender with TLS and connects to the specified host and port. + * Uses synchronous mode for backward compatibility. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @return connected sender + */ + public static IlpV4WebSocketSender connect(String host, int port, boolean tlsEnabled) { + IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, + 0, 0, 0, // No auto-flush in sync mode + 1, 1 // window=1 for sync behavior, queue=1 (not used) + ); + sender.ensureConnected(); + return sender; + } + + /** + * Creates a new sender with async mode and custom configuration. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @return connected sender + */ + public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos) { + return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + DEFAULT_IN_FLIGHT_WINDOW_SIZE, DEFAULT_SEND_QUEUE_CAPACITY); + } + + /** + * Creates a new sender with async mode and full configuration including flow control. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize max batches awaiting server ACK (default: 8) + * @param sendQueueCapacity max batches waiting to send (default: 16) + * @return connected sender + */ + public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, int sendQueueCapacity) { + IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, sendQueueCapacity + ); + sender.ensureConnected(); + return sender; + } + + /** + * Creates a new sender with async mode and default configuration. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @return connected sender + */ + public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { + return connectAsync(host, port, tlsEnabled, + DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS); + } + + /** + * Factory method for SenderBuilder integration. + */ + public static IlpV4WebSocketSender create( + String host, + int port, + boolean tlsEnabled, + int bufferSize, + String authToken, + String username, + String password + ) { + IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + host, port, tlsEnabled, bufferSize, + 0, 0, 0, + 1, 1 // window=1 for sync behavior + ); + // TODO: Store auth credentials for connection + sender.ensureConnected(); + return sender; + } + + /** + * Creates a sender without connecting. For testing only. + *

+ * This allows unit tests to test sender logic without requiring a real server. + * + * @param host server host (not connected) + * @param port server port (not connected) + * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async + * @return unconnected sender + */ + public static IlpV4WebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { + return new IlpV4WebSocketSender( + host, port, false, DEFAULT_BUFFER_SIZE, + 0, 0, 0, + inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY + ); + // Note: does NOT call ensureConnected() + } + + /** + * Creates a sender with custom flow control settings without connecting. For testing only. + * + * @param host server host (not connected) + * @param port server port (not connected) + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async + * @param sendQueueCapacity max batches waiting to send + * @return unconnected sender + */ + public static IlpV4WebSocketSender createForTesting( + String host, int port, + int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, + int inFlightWindowSize, int sendQueueCapacity) { + return new IlpV4WebSocketSender( + host, port, false, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, sendQueueCapacity + ); + // Note: does NOT call ensureConnected() + } + + private void ensureConnected() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + if (!connected) { + // Create WebSocket client using factory (zero-GC native implementation) + if (tlsEnabled) { + client = WebSocketClientFactory.newInsecureTlsInstance(); + } else { + client = WebSocketClientFactory.newPlainTextInstance(); + } + + // Connect and upgrade to WebSocket + try { + client.connect(host, port); + client.upgrade(WRITE_PATH); + } catch (Exception e) { + client.close(); + client = null; + throw new LineSenderException("Failed to connect to " + host + ":" + port, e); + } + + // a window for tracking batches awaiting ACK (both modes) + inFlightWindow = new InFlightWindow(inFlightWindowSize, InFlightWindow.DEFAULT_TIMEOUT_MS); + + // Initialize send queue for async mode (window > 1) + // The send queue handles both sending AND receiving (single I/O thread) + if (inFlightWindowSize > 1) { + sendQueue = new WebSocketSendQueue(client, inFlightWindow, + sendQueueCapacity, + WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, + WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + // Sync mode (window=1): no send queue - we send and read ACKs synchronously + + // Clear sent schema hashes - server starts fresh on each connection + sentSchemaHashes.clear(); + + connected = true; + LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}]", host, port, inFlightWindowSize); + } + } + + /** + * Returns whether Gorilla encoding is enabled. + */ + public boolean isGorillaEnabled() { + return gorillaEnabled; + } + + /** + * Sets whether to use Gorilla timestamp encoding. + */ + public IlpV4WebSocketSender setGorillaEnabled(boolean enabled) { + this.gorillaEnabled = enabled; + this.encoder.setGorillaEnabled(enabled); + return this; + } + + /** + * Returns whether async mode is enabled (window size > 1). + */ + public boolean isAsyncMode() { + return inFlightWindowSize > 1; + } + + /** + * Returns the in-flight window size. + * Window=1 means sync mode, window>1 means async mode. + */ + public int getInFlightWindowSize() { + return inFlightWindowSize; + } + + /** + * Returns the send queue capacity. + */ + public int getSendQueueCapacity() { + return sendQueueCapacity; + } + + /** + * Returns the auto-flush row threshold. + */ + public int getAutoFlushRows() { + return autoFlushRows; + } + + /** + * Returns the auto-flush byte threshold. + */ + public int getAutoFlushBytes() { + return autoFlushBytes; + } + + /** + * Returns the auto-flush interval in nanoseconds. + */ + public long getAutoFlushIntervalNanos() { + return autoFlushIntervalNanos; + } + + /** + * Returns the global symbol dictionary. + * For testing and encoder integration. + */ + public GlobalSymbolDictionary getGlobalSymbolDictionary() { + return globalSymbolDictionary; + } + + /** + * Returns the max symbol ID sent to the server. + * Once sent over TCP, server is guaranteed to receive it (or connection dies). + */ + public int getMaxSentSymbolId() { + return maxSentSymbolId; + } + + // ==================== Fast-path API for high-throughput generators ==================== + // + // These methods bypass the normal fluent API to avoid per-row overhead: + // - No hashmap lookups for column names + // - No checkNotClosed()/checkTableSelected() per column + // - Direct access to column buffers + // + // Usage: + // // Setup (once) + // IlpV4TableBuffer tableBuffer = sender.getTableBuffer("q"); + // IlpV4TableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); + // IlpV4TableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); + // + // // Hot path (per row) + // colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol)); + // colBid.addDouble(bid); + // tableBuffer.nextRow(); + // sender.incrementPendingRowCount(); + + /** + * Gets or creates a table buffer for direct access. + * For high-throughput generators that want to bypass fluent API overhead. + */ + public IlpV4TableBuffer getTableBuffer(String tableName) { + IlpV4TableBuffer buffer = tableBuffers.get(tableName); + if (buffer == null) { + buffer = new IlpV4TableBuffer(tableName); + tableBuffers.put(tableName, buffer); + } + currentTableBuffer = buffer; + currentTableName = tableName; + return buffer; + } + + /** + * Registers a symbol in the global dictionary and returns its ID. + * For use with fast-path column buffer access. + */ + public int getOrAddGlobalSymbol(String value) { + int globalId = globalSymbolDictionary.getOrAddSymbol(value); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + return globalId; + } + + /** + * Increments the pending row count for auto-flush tracking. + * Call this after adding a complete row via fast-path API. + * Triggers auto-flush if any threshold is exceeded. + */ + public void incrementPendingRowCount() { + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; + + // Check if any flush threshold is exceeded (same as sendRow()) + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); + } + } + } + + // ==================== Sender interface implementation ==================== + + @Override + public IlpV4WebSocketSender table(CharSequence tableName) { + checkNotClosed(); + // Fast path: if table name matches current, skip hashmap lookup + if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { + return this; + } + // Table changed - invalidate cached column references + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableName = tableName.toString(); + currentTableBuffer = tableBuffers.get(currentTableName); + if (currentTableBuffer == null) { + currentTableBuffer = new IlpV4TableBuffer(currentTableName); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Both modes accumulate rows until flush + return this; + } + + @Override + public IlpV4WebSocketSender symbol(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + + if (value != null) { + // Register symbol in global dictionary and track max ID for delta calculation + String symbolValue = value.toString(); + int globalId = globalSymbolDictionary.getOrAddSymbol(symbolValue); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + // Store global ID in the column buffer + col.addSymbolWithGlobalId(symbolValue, globalId); + } else { + col.addSymbol(null); + } + return this; + } + + @Override + public IlpV4WebSocketSender boolColumn(CharSequence columnName, boolean value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + col.addBoolean(value); + return this; + } + + @Override + public IlpV4WebSocketSender longColumn(CharSequence columnName, long value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + col.addLong(value); + return this; + } + + /** + * Adds an INT column value to the current row. + * + * @param columnName the column name + * @param value the int value + * @return this sender for method chaining + */ + public IlpV4WebSocketSender intColumn(CharSequence columnName, int value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); + col.addInt(value); + return this; + } + + @Override + public IlpV4WebSocketSender doubleColumn(CharSequence columnName, double value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); + col.addDouble(value); + return this; + } + + @Override + public IlpV4WebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + col.addString(value != null ? value.toString() : null); + return this; + } + + /** + * Adds a SHORT column value to the current row. + * + * @param columnName the column name + * @param value the short value + * @return this sender for method chaining + */ + public IlpV4WebSocketSender shortColumn(CharSequence columnName, short value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); + col.addShort(value); + return this; + } + + /** + * Adds a CHAR column value to the current row. + *

+ * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB. + * + * @param columnName the column name + * @param value the character value + * @return this sender for method chaining + */ + public IlpV4WebSocketSender charColumn(CharSequence columnName, char value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); + col.addShort((short) value); + return this; + } + + /** + * Adds a UUID column value to the current row. + * + * @param columnName the column name + * @param lo the low 64 bits of the UUID + * @param hi the high 64 bits of the UUID + * @return this sender for method chaining + */ + public IlpV4WebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); + col.addUuid(hi, lo); + return this; + } + + /** + * Adds a LONG256 column value to the current row. + * + * @param columnName the column name + * @param l0 the least significant 64 bits + * @param l1 the second 64 bits + * @param l2 the third 64 bits + * @param l3 the most significant 64 bits + * @return this sender for method chaining + */ + public IlpV4WebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG256, true); + col.addLong256(l0, l1, l2, l3); + return this; + } + + @Override + public IlpV4WebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); + col.addLong(value); + } else { + long micros = toMicros(value, unit); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + col.addLong(micros); + } + return this; + } + + @Override + public IlpV4WebSocketSender timestampColumn(CharSequence columnName, Instant value) { + checkNotClosed(); + checkTableSelected(); + long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + col.addLong(micros); + return this; + } + + @Override + public void at(long timestamp, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + atNanos(timestamp); + } else { + long micros = toMicros(timestamp, unit); + atMicros(micros); + } + } + + @Override + public void at(Instant timestamp) { + checkNotClosed(); + checkTableSelected(); + long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; + atMicros(micros); + } + + private void atMicros(long timestampMicros) { + // Add designated timestamp column (empty name for designated timestamp) + // Use cached reference to avoid hashmap lookup per row + if (cachedTimestampColumn == null) { + cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + } + cachedTimestampColumn.addLong(timestampMicros); + sendRow(); + } + + private void atNanos(long timestampNanos) { + // Add designated timestamp column (empty name for designated timestamp) + // Use cached reference to avoid hashmap lookup per row + if (cachedTimestampNanosColumn == null) { + cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); + } + cachedTimestampNanosColumn.addLong(timestampNanos); + sendRow(); + } + + @Override + public void atNow() { + checkNotClosed(); + checkTableSelected(); + // Server-assigned timestamp - just send the row without designated timestamp + sendRow(); + } + + /** + * Accumulates the current row. + * Both sync and async modes buffer rows until flush (explicit or auto-flush). + * The difference is that sync mode flush() blocks until server ACKs. + */ + private void sendRow() { + ensureConnected(); + currentTableBuffer.nextRow(); + + // Both modes: accumulate rows, don't encode yet + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; + + // Check if any flush threshold is exceeded + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); + } + } + } + + /** + * Checks if any auto-flush threshold is exceeded. + */ + private boolean shouldAutoFlush() { + if (pendingRowCount <= 0) { + return false; + } + // Row limit + if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { + return true; + } + // Time limit + if (autoFlushIntervalNanos > 0) { + long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; + if (ageNanos >= autoFlushIntervalNanos) { + return true; + } + } + // Byte limit is harder to estimate without encoding, skip for now + return false; + } + + /** + * Flushes pending rows by encoding and sending them. + * Each table's rows are encoded into a separate ILP v4 message and sent as one WebSocket frame. + */ + private void flushPendingRows() { + if (pendingRowCount <= 0) { + return; + } + + LOG.debug("Flushing pending rows [count={}, tables={}]", pendingRowCount, tableBuffers.size()); + + // Ensure activeBuffer is ready for writing + // It might be in RECYCLED state if previous batch was sent but we didn't swap yet + ensureActiveBufferReady(); + + // Encode all table buffers that have data + // Iterate over the keys list directly + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; // Skip null entries (shouldn't happen but be safe) + } + IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null) { + continue; + } + int rowCount = tableBuffer.getRowCount(); + if (rowCount > 0) { + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); + + LOG.debug("Encoding table [name={}, rows={}, maxSentSymbolId={}, batchMaxId={}, useSchemaRef={}]", tableName, rowCount, maxSentSymbolId, currentBatchMaxSymbolId, useSchemaRef); + + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); + + // Track schema key if this was the first time sending this schema + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + IlpBufferWriter buffer = encoder.getBuffer(); + + // Copy to microbatch buffer and seal immediately + // Each ILP v4 message must be in its own WebSocket frame + activeBuffer.ensureCapacity(messageSize); + activeBuffer.write(buffer.getBufferPtr(), messageSize); + activeBuffer.incrementRowCount(); + activeBuffer.setMaxSymbolId(currentBatchMaxSymbolId); + + // Update maxSentSymbolId - once sent over TCP, server will receive it + maxSentSymbolId = currentBatchMaxSymbolId; + + // Seal and enqueue for sending + sealAndSwapBuffer(); + + // Reset table buffer and batch-level symbol tracking + tableBuffer.reset(); + currentBatchMaxSymbolId = -1; + } + } + + // Reset pending count + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + } + + /** + * Ensures the active buffer is ready for writing (in FILLING state). + * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. + */ + private void ensureActiveBufferReady() { + if (activeBuffer.isFilling()) { + return; // Already ready + } + + if (activeBuffer.isRecycled()) { + // Buffer was recycled but not reset - reset it now + activeBuffer.reset(); + return; + } + + // Buffer is in use (SEALED or SENDING) - wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for active buffer to be recycled"); + } + } + + // Buffer should now be RECYCLED - reset it + if (activeBuffer.isRecycled()) { + activeBuffer.reset(); + } + } + + /** + * Adds encoded data to the active microbatch buffer. + * Triggers seal and swap if buffer is full. + */ + private void addToMicrobatch(long dataPtr, int length) { + // Ensure activeBuffer is ready for writing + ensureActiveBufferReady(); + + // If current buffer can't hold the data, seal and swap + if (activeBuffer.hasData() && + activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { + sealAndSwapBuffer(); + } + + // Ensure buffer can hold the data + activeBuffer.ensureCapacity(activeBuffer.getBufferPos() + length); + + // Copy data to buffer + activeBuffer.write(dataPtr, length); + activeBuffer.incrementRowCount(); + } + + /** + * Seals the current buffer and swaps to the other buffer. + * Enqueues the sealed buffer for async sending. + */ + private void sealAndSwapBuffer() { + if (!activeBuffer.hasData()) { + return; // Nothing to send + } + + MicrobatchBuffer toSend = activeBuffer; + toSend.seal(); + + LOG.debug("Sealing buffer [id={}, rows={}, bytes={}]", toSend.getBatchId(), toSend.getRowCount(), toSend.getBufferPos()); + + // Swap to the other buffer + activeBuffer = (activeBuffer == buffer0) ? buffer1 : buffer0; + + // If the other buffer is still being sent, wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + LOG.debug("Waiting for buffer recycle [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for buffer to be recycled"); + } + LOG.debug("Buffer recycled [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } + + // Reset the new active buffer + int stateBeforeReset = activeBuffer.getState(); + LOG.debug("Resetting buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(stateBeforeReset)); + activeBuffer.reset(); + + // Enqueue the sealed buffer for sending. + // If enqueue fails, roll back local state so the same batch can be retried. + try { + if (!sendQueue.enqueue(toSend)) { + throw new LineSenderException("Failed to enqueue buffer for sending"); + } + } catch (LineSenderException e) { + activeBuffer = toSend; + if (toSend.isSealed()) { + toSend.rollbackSealForRetry(); + } + throw e; + } + } + + @Override + public void flush() { + checkNotClosed(); + ensureConnected(); + + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush pending rows and wait for ACKs + flushPendingRows(); + + // Flush any remaining data in the active microbatch buffer + if (activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + + // Wait for all pending batches to be sent to the server + sendQueue.flush(); + + // Wait for all in-flight batches to be acknowledged by the server + inFlightWindow.awaitEmpty(); + + LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); + } else { + // Sync mode (window=1): flush pending rows and wait for ACKs synchronously + flushSync(); + } + } + + /** + * Flushes pending rows synchronously, blocking until server ACKs. + * Used in sync mode for simpler, blocking operation. + */ + private void flushSync() { + if (pendingRowCount <= 0) { + return; + } + + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); + + // Encode all table buffers that have data into a single message + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; + } + IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); + + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); + + // Track schema key if this was the first time sending this schema + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + + if (messageSize > 0) { + IlpBufferWriter buffer = encoder.getBuffer(); + + // Track batch in InFlightWindow before sending + long batchSequence = nextBatchSequence++; + inFlightWindow.addInFlight(batchSequence); + + // Update maxSentSymbolId - once sent over TCP, server will receive it + maxSentSymbolId = currentBatchMaxSymbolId; + + LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef); + + // Send over WebSocket + client.sendBinary(buffer.getBufferPtr(), messageSize); + + // Wait for ACK synchronously + waitForAck(batchSequence); + } + + // Reset table buffer after sending + tableBuffer.reset(); + + // Reset batch-level symbol tracking + currentBatchMaxSymbolId = -1; + } + + // Reset pending row tracking + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + + LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); + } + + /** + * Waits synchronously for an ACK from the server for the specified batch. + */ + private void waitForAck(long expectedSequence) { + WebSocketResponse response = new WebSocketResponse(); + long deadline = System.currentTimeMillis() + InFlightWindow.DEFAULT_TIMEOUT_MS; + + while (System.currentTimeMillis() < deadline) { + try { + final boolean[] sawBinary = {false}; + boolean received = client.receiveFrame(new WebSocketFrameHandler() { + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + sawBinary[0] = true; + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + throw new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + } + if (!response.readFrom(payloadPtr, payloadLen)) { + throw new LineSenderException("Failed to parse ACK response"); + } + } + + @Override + public void onClose(int code, String reason) { + throw new LineSenderException("WebSocket closed while waiting for ACK: " + reason); + } + }, 1000); // 1 second timeout per read attempt + + if (received) { + // Non-binary frames (e.g. ping/pong/text) are not ACKs. + if (!sawBinary[0]) { + continue; + } + long sequence = response.getSequence(); + if (response.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + inFlightWindow.acknowledgeUpTo(sequence); + if (sequence >= expectedSequence) { + return; // Our batch was acknowledged (cumulative) + } + // Got ACK for lower sequence - continue waiting + } else { + String errorMessage = response.getErrorMessage(); + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + response.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + if (sequence == expectedSequence) { + throw error; + } + } + } + } catch (LineSenderException e) { + failExpectedIfNeeded(expectedSequence, e); + throw e; + } catch (Exception e) { + LineSenderException wrapped = new LineSenderException("Error waiting for ACK: " + e.getMessage(), e); + failExpectedIfNeeded(expectedSequence, wrapped); + throw wrapped; + } + } + + LineSenderException timeout = new LineSenderException("Timeout waiting for ACK for batch " + expectedSequence); + failExpectedIfNeeded(expectedSequence, timeout); + throw timeout; + } + + private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { + if (inFlightWindow != null && inFlightWindow.getLastError() == null) { + inFlightWindow.fail(expectedSequence, error); + } + } + + @Override + public DirectByteSlice bufferView() { + throw new LineSenderException("bufferView() is not supported for WebSocket sender"); + } + + @Override + public void cancelRow() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + } + } + + @Override + public void reset() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.reset(); + } + } + + // ==================== Array methods ==================== + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(CharSequence name, DoubleArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(array); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(CharSequence name, LongArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(array); + return this; + } + + // ==================== Decimal methods ==================== + + @Override + public Sender decimalColumn(CharSequence name, Decimal64 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + col.addDecimal64(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal128 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + col.addDecimal128(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal256 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, CharSequence value) { + if (value == null || value.length() == 0) return this; + checkNotClosed(); + checkTableSelected(); + try { + java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); + Decimal256 decimal = Decimal256.fromBigDecimal(bd); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(decimal); + } catch (Exception e) { + throw new LineSenderException("Failed to parse decimal value: " + value, e); + } + return this; + } + + // ==================== Helper methods ==================== + + private long toMicros(long value, ChronoUnit unit) { + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); + } + } + + private void checkNotClosed() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + } + + private void checkTableSelected() { + if (currentTableBuffer == null) { + throw new LineSenderException("table() must be called before adding columns"); + } + } + + @Override + public void close() { + if (!closed) { + closed = true; + + // Flush any remaining data + try { + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush accumulated rows in table buffers first + flushPendingRows(); + + if (activeBuffer != null && activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + if (sendQueue != null) { + sendQueue.close(); + } + } else { + // Sync mode (window=1): flush pending rows synchronously + if (pendingRowCount > 0 && client != null && client.isConnected()) { + flushSync(); + } + } + } catch (Exception e) { + LOG.error("Error during close: {}", String.valueOf(e)); + } + + // Close buffers (async mode only, window > 1) + if (buffer0 != null) { + buffer0.close(); + } + if (buffer1 != null) { + buffer1.close(); + } + + if (client != null) { + client.close(); + client = null; + } + encoder.close(); + tableBuffers.clear(); + + LOG.info("IlpV4WebSocketSender closed"); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java new file mode 100644 index 0000000..9db5456 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java @@ -0,0 +1,468 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +/** + * Lock-free in-flight batch tracker for the sliding window protocol. + *

+ * Concurrency model (lock-free): + *

+ * Assumptions that keep it simple and lock-free: + * + * With these constraints we can rely on volatile reads/writes (no CAS) and still + * offer blocking waits for space/empty without protecting the counters with locks. + */ +public class InFlightWindow { + + private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class); + + public static final int DEFAULT_WINDOW_SIZE = 8; + public static final long DEFAULT_TIMEOUT_MS = 30_000; + + // Spin parameters + private static final int SPIN_TRIES = 100; + private static final long PARK_NANOS = 100_000; // 100 microseconds + + private final int maxWindowSize; + private final long timeoutMs; + + // Core state + // highestSent: the sequence number of the last batch added to the window + private volatile long highestSent = -1; + + // highestAcked: the sequence number of the last acknowledged batch (cumulative) + private volatile long highestAcked = -1; + + // Error state + private final AtomicReference lastError = new AtomicReference<>(); + private volatile long failedBatchId = -1; + + // Thread waiting for space (sender thread) + private volatile Thread waitingForSpace; + + // Thread waiting for empty (flush thread) + private volatile Thread waitingForEmpty; + + // Statistics (not strictly accurate under contention, but good enough for monitoring) + private volatile long totalAcked = 0; + private volatile long totalFailed = 0; + + /** + * Creates a new InFlightWindow with default configuration. + */ + public InFlightWindow() { + this(DEFAULT_WINDOW_SIZE, DEFAULT_TIMEOUT_MS); + } + + /** + * Creates a new InFlightWindow with custom configuration. + * + * @param maxWindowSize maximum number of batches in flight + * @param timeoutMs timeout for blocking operations + */ + public InFlightWindow(int maxWindowSize, long timeoutMs) { + if (maxWindowSize <= 0) { + throw new IllegalArgumentException("maxWindowSize must be positive"); + } + this.maxWindowSize = maxWindowSize; + this.timeoutMs = timeoutMs; + } + + /** + * Checks if there's space in the window for another batch. + * Wait-free operation. + * + * @return true if there's space, false if window is full + */ + public boolean hasWindowSpace() { + return getInFlightCount() < maxWindowSize; + } + + /** + * Tries to add a batch to the in-flight window without blocking. + * Lock-free, assuming single producer for highestSent. + * + * Called by: async producer (WebSocket I/O thread) before sending a batch. + * @param batchId the batch ID to track (must be sequential) + * @return true if added, false if window is full + */ + public boolean tryAddInFlight(long batchId) { + // Check window space first + long sent = highestSent; + long acked = highestAcked; + + if (sent - acked >= maxWindowSize) { + return false; + } + + // Sequential caller: just publish the new highestSent + highestSent = batchId; + + LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount()); + return true; + } + + /** + * Adds a batch to the in-flight window. + *

+ * Blocks if the window is full until space becomes available or timeout. + * Uses spin-wait with exponential backoff, then parks. Blocking is only expected + * in modes where another actor can make progress on acknowledgments. In normal + * sync usage the window size is 1 and the same thread immediately waits for the + * ACK, so this should never actually park. If a caller uses a larger window here + * it must ensure ACKs are processed on another thread; a single-threaded caller + * with window>1 would deadlock by parking while also being the only thread that + * can advance {@link #acknowledgeUpTo(long)}. + * + * Called by: sync sender thread before sending a batch (window=1). + * @param batchId the batch ID to track + * @throws LineSenderException if timeout occurs or an error was reported + */ + public void addInFlight(long batchId) { + // Check for errors first + checkError(); + + // Fast path: try to add without waiting + if (tryAddInFlightInternal(batchId)) { + return; + } + + // Slow path: need to wait for space + long deadline = System.currentTimeMillis() + timeoutMs; + int spins = 0; + + // Register as waiting thread + waitingForSpace = Thread.currentThread(); + try { + while (true) { + // Check for errors + checkError(); + + // Try to add + if (tryAddInFlightInternal(batchId)) { + return; + } + + // Check timeout + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Timeout waiting for window space, window full with " + + getInFlightCount() + " batches"); + } + + // Spin or park + if (spins < SPIN_TRIES) { + Thread.onSpinWait(); + spins++; + } else { + // Park with timeout + LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000)); + if (Thread.interrupted()) { + throw new LineSenderException("Interrupted while waiting for window space"); + } + } + } + } finally { + waitingForSpace = null; + } + } + + private boolean tryAddInFlightInternal(long batchId) { + long sent = highestSent; + long acked = highestAcked; + + if (sent - acked >= maxWindowSize) { + return false; + } + + // For sequential IDs, we just update highestSent + // The caller guarantees batchId is the next in sequence + highestSent = batchId; + + LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount()); + return true; + } + + /** + * Acknowledges a batch, removing it from the in-flight window. + *

+ * For sequential batch IDs, this is a cumulative acknowledgment - + * acknowledging batch N means all batches up to N are acknowledged. + * + * Called by: acker (WebSocket I/O thread) after receiving an ACK. + * @param batchId the batch ID that was acknowledged + * @return true if the batch was in flight, false if already acknowledged + */ + public boolean acknowledge(long batchId) { + return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId; + } + + /** + * Acknowledges all batches up to and including the given sequence (cumulative ACK). + * Lock-free with single consumer. + * + * Called by: acker (WebSocket I/O thread) after receiving an ACK. + * @param sequence the highest acknowledged sequence + * @return the number of batches acknowledged + */ + public int acknowledgeUpTo(long sequence) { + long sent = highestSent; + + // Nothing to acknowledge if window is empty or sequence is beyond what's sent + if (sent < 0) { + return 0; // No batches have been sent + } + + // Cap sequence at highestSent - can't acknowledge what hasn't been sent + long effectiveSequence = Math.min(sequence, sent); + + long prevAcked = highestAcked; + if (effectiveSequence <= prevAcked) { + // Already acknowledged up to this point + return 0; + } + highestAcked = effectiveSequence; + + int acknowledged = (int) (effectiveSequence - prevAcked); + totalAcked += acknowledged; + + LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount()); + + // Wake up waiting threads + Thread waiter = waitingForSpace; + if (waiter != null) { + LockSupport.unpark(waiter); + } + + waiter = waitingForEmpty; + if (waiter != null && getInFlightCount() == 0) { + LockSupport.unpark(waiter); + } + + return acknowledged; + } + + /** + * Marks a batch as failed, setting an error that will be propagated to waiters. + * + * Called by: acker (WebSocket I/O thread) on error response or send failure. + * @param batchId the batch ID that failed + * @param error the error that occurred + */ + public void fail(long batchId, Throwable error) { + this.failedBatchId = batchId; + this.lastError.set(error); + totalFailed++; + + LOG.error("Batch failed [batchId={}, error={}]", batchId, String.valueOf(error)); + + wakeWaiters(); + } + + /** + * Marks all currently in-flight batches as failed. + *

+ * Used for transport-level failures (disconnect/protocol violation) where + * no further ACKs are expected and all waiters must be released. + * + * @param error terminal error to propagate + */ + public void failAll(Throwable error) { + long sent = highestSent; + long acked = highestAcked; + long inFlight = Math.max(0, sent - acked); + + this.failedBatchId = sent; + this.lastError.set(error); + totalFailed += Math.max(1, inFlight); + + LOG.error("All in-flight batches failed [inFlight={}, error={}]", inFlight, String.valueOf(error)); + + wakeWaiters(); + } + + /** + * Waits until all in-flight batches are acknowledged. + *

+ * Called by flush() to ensure all data is confirmed. + * + * Called by: waiter (flush thread), while producer/acker thread progresses. + * @throws LineSenderException if timeout occurs or an error was reported + */ + public void awaitEmpty() { + checkError(); + + // Fast path: already empty + if (getInFlightCount() == 0) { + LOG.debug("Window already empty"); + return; + } + + long deadline = System.currentTimeMillis() + timeoutMs; + int spins = 0; + + // Register as waiting thread + waitingForEmpty = Thread.currentThread(); + try { + while (getInFlightCount() > 0) { + checkError(); + + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Timeout waiting for batch acknowledgments, " + + getInFlightCount() + " batches still in flight"); + } + + if (spins < SPIN_TRIES) { + Thread.onSpinWait(); + spins++; + } else { + LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000)); + if (Thread.interrupted()) { + throw new LineSenderException("Interrupted while waiting for acknowledgments"); + } + } + } + + LOG.debug("Window empty, all batches ACKed"); + } finally { + waitingForEmpty = null; + } + } + + /** + * Returns the current number of batches in flight. + * Wait-free operation. + */ + public int getInFlightCount() { + long sent = highestSent; + long acked = highestAcked; + // Ensure non-negative (can happen during initialization) + return (int) Math.max(0, sent - acked); + } + + /** + * Returns true if the window is empty. + * Wait-free operation. + */ + public boolean isEmpty() { + return getInFlightCount() == 0; + } + + /** + * Returns true if the window is full. + * Wait-free operation. + */ + public boolean isFull() { + return getInFlightCount() >= maxWindowSize; + } + + /** + * Returns the maximum window size. + */ + public int getMaxWindowSize() { + return maxWindowSize; + } + + /** + * Returns the total number of batches acknowledged. + */ + public long getTotalAcked() { + return totalAcked; + } + + /** + * Returns the total number of batches that failed. + */ + public long getTotalFailed() { + return totalFailed; + } + + /** + * Returns the last error, or null if no error. + */ + public Throwable getLastError() { + return lastError.get(); + } + + /** + * Clears the error state. + */ + public void clearError() { + lastError.set(null); + failedBatchId = -1; + } + + /** + * Resets the window, clearing all state. + */ + public void reset() { + highestSent = -1; + highestAcked = -1; + lastError.set(null); + failedBatchId = -1; + + wakeWaiters(); + } + + private void checkError() { + Throwable error = lastError.get(); + if (error != null) { + throw new LineSenderException("Batch " + failedBatchId + " failed: " + error.getMessage(), error); + } + } + + private void wakeWaiters() { + Thread waiter = waitingForSpace; + if (waiter != null) { + LockSupport.unpark(waiter); + } + waiter = waitingForEmpty; + if (waiter != null) { + LockSupport.unpark(waiter); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java new file mode 100644 index 0000000..832bb66 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java @@ -0,0 +1,501 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * A buffer for accumulating ILP data into microbatches before sending. + *

+ * This class implements a state machine for buffer lifecycle management in the + * double-buffering scheme used by {@link IlpV4WebSocketSender}: + * 

+ * Buffer States:
+ * ┌─────────────┐    seal()     ┌─────────────┐    markSending()  ┌─────────────┐
+ * │   FILLING   │──────────────►│   SEALED    │──────────────────►│   SENDING   │
+ * │ (user owns) │               │ (in queue)  │                   │ (I/O owns)  │
+ * └─────────────┘               └─────────────┘                   └──────┬──────┘
+ *        ▲                                                               │
+ *        │                         markRecycled()                        │
+ *        └───────────────────────────────────────────────────────────────┘
+ *                              (after send complete)
+ *
+ *

+ * Thread safety: This class is NOT thread-safe for concurrent writes. However, it + * supports safe hand-over between user thread and I/O thread through the state + * machine. State transitions use volatile fields to ensure visibility. + */ +public class MicrobatchBuffer implements QuietCloseable { + + // Buffer states + public static final int STATE_FILLING = 0; + public static final int STATE_SEALED = 1; + public static final int STATE_SENDING = 2; + public static final int STATE_RECYCLED = 3; + + // Flush trigger thresholds + private final int maxRows; + private final int maxBytes; + private final long maxAgeNanos; + + // Native memory buffer + private long bufferPtr; + private int bufferCapacity; + private int bufferPos; + + // Row tracking + private int rowCount; + private long firstRowTimeNanos; + + // Symbol tracking for delta encoding + private int maxSymbolId = -1; + + // Batch identification + private long batchId; + private static long nextBatchId = 0; + + // State machine + private volatile int state = STATE_FILLING; + + // For waiting on recycle (user thread waits for I/O thread to finish) + // CountDownLatch is not resettable, so we create a new instance on reset() + private volatile CountDownLatch recycleLatch = new CountDownLatch(1); + + /** + * Creates a new MicrobatchBuffer with specified flush thresholds. + * + * @param initialCapacity initial buffer size in bytes + * @param maxRows maximum rows before auto-flush (0 = unlimited) + * @param maxBytes maximum bytes before auto-flush (0 = unlimited) + * @param maxAgeNanos maximum age in nanoseconds before auto-flush (0 = unlimited) + */ + public MicrobatchBuffer(int initialCapacity, int maxRows, int maxBytes, long maxAgeNanos) { + if (initialCapacity <= 0) { + throw new IllegalArgumentException("initialCapacity must be positive"); + } + this.bufferCapacity = initialCapacity; + this.bufferPtr = Unsafe.malloc(initialCapacity, MemoryTag.NATIVE_ILP_RSS); + this.bufferPos = 0; + this.rowCount = 0; + this.firstRowTimeNanos = 0; + this.maxRows = maxRows; + this.maxBytes = maxBytes; + this.maxAgeNanos = maxAgeNanos; + this.batchId = nextBatchId++; + } + + /** + * Creates a new MicrobatchBuffer with default thresholds (no auto-flush). + * + * @param initialCapacity initial buffer size in bytes + */ + public MicrobatchBuffer(int initialCapacity) { + this(initialCapacity, 0, 0, 0); + } + + // ==================== DATA OPERATIONS ==================== + + /** + * Returns the buffer pointer for writing data. + * Only valid when state is FILLING. + */ + public long getBufferPtr() { + return bufferPtr; + } + + /** + * Returns the current write position in the buffer. + */ + public int getBufferPos() { + return bufferPos; + } + + /** + * Returns the buffer capacity. + */ + public int getBufferCapacity() { + return bufferCapacity; + } + + /** + * Sets the buffer position after external writes. + * Only valid when state is FILLING. + * + * @param pos new position + */ + public void setBufferPos(int pos) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot set position when state is " + stateName(state)); + } + if (pos < 0 || pos > bufferCapacity) { + throw new IllegalArgumentException("Position out of bounds: " + pos); + } + this.bufferPos = pos; + } + + /** + * Ensures the buffer has at least the specified capacity. + * Grows the buffer if necessary. + * + * @param requiredCapacity minimum required capacity + */ + public void ensureCapacity(int requiredCapacity) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot resize when state is " + stateName(state)); + } + if (requiredCapacity > bufferCapacity) { + int newCapacity = Math.max(bufferCapacity * 2, requiredCapacity); + long newPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); + bufferPtr = newPtr; + bufferCapacity = newCapacity; + } + } + + /** + * Writes bytes to the buffer at the current position. + * Grows the buffer if necessary. + * + * @param src source address + * @param length number of bytes to write + */ + public void write(long src, int length) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot write when state is " + stateName(state)); + } + ensureCapacity(bufferPos + length); + Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length); + bufferPos += length; + } + + /** + * Writes a single byte to the buffer. + * + * @param b byte to write + */ + public void writeByte(byte b) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot write when state is " + stateName(state)); + } + ensureCapacity(bufferPos + 1); + Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b); + bufferPos++; + } + + /** + * Increments the row count and records the first row time if this is the first row. + */ + public void incrementRowCount() { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot increment row count when state is " + stateName(state)); + } + if (rowCount == 0) { + firstRowTimeNanos = System.nanoTime(); + } + rowCount++; + } + + /** + * Returns the number of rows in this buffer. + */ + public int getRowCount() { + return rowCount; + } + + /** + * Returns true if the buffer has any data. + */ + public boolean hasData() { + return bufferPos > 0; + } + + /** + * Returns the batch ID for this buffer. + */ + public long getBatchId() { + return batchId; + } + + /** + * Returns the maximum symbol ID used in this batch. + * Used for delta symbol dictionary tracking. + */ + public int getMaxSymbolId() { + return maxSymbolId; + } + + /** + * Sets the maximum symbol ID used in this batch. + * Used for delta symbol dictionary tracking. + */ + public void setMaxSymbolId(int maxSymbolId) { + this.maxSymbolId = maxSymbolId; + } + + // ==================== FLUSH TRIGGER CHECKS ==================== + + /** + * Checks if the buffer should be flushed based on configured thresholds. + * + * @return true if any flush threshold is exceeded + */ + public boolean shouldFlush() { + if (!hasData()) { + return false; + } + return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded(); + } + + /** + * Checks if the row count limit has been exceeded. + */ + public boolean isRowLimitExceeded() { + return maxRows > 0 && rowCount >= maxRows; + } + + /** + * Checks if the byte size limit has been exceeded. + */ + public boolean isByteLimitExceeded() { + return maxBytes > 0 && bufferPos >= maxBytes; + } + + /** + * Checks if the age limit has been exceeded. + */ + public boolean isAgeLimitExceeded() { + if (maxAgeNanos <= 0 || rowCount == 0) { + return false; + } + long ageNanos = System.nanoTime() - firstRowTimeNanos; + return ageNanos >= maxAgeNanos; + } + + /** + * Returns the age of the first row in nanoseconds, or 0 if no rows. + */ + public long getAgeNanos() { + if (rowCount == 0) { + return 0; + } + return System.nanoTime() - firstRowTimeNanos; + } + + // ==================== STATE MACHINE ==================== + + /** + * Returns the current state. + */ + public int getState() { + return state; + } + + /** + * Returns true if the buffer is in FILLING state (available for writing). + */ + public boolean isFilling() { + return state == STATE_FILLING; + } + + /** + * Returns true if the buffer is in SEALED state (ready to send). + */ + public boolean isSealed() { + return state == STATE_SEALED; + } + + /** + * Returns true if the buffer is in SENDING state (being sent by I/O thread). + */ + public boolean isSending() { + return state == STATE_SENDING; + } + + /** + * Returns true if the buffer is in RECYCLED state (available for reset). + */ + public boolean isRecycled() { + return state == STATE_RECYCLED; + } + + /** + * Returns true if the buffer is currently in use (not available for the user thread). + */ + public boolean isInUse() { + int s = state; + return s == STATE_SEALED || s == STATE_SENDING; + } + + /** + * Seals the buffer, transitioning from FILLING to SEALED. + * After sealing, no more data can be written. + * Only the user thread should call this. + * + * @throws IllegalStateException if not in FILLING state + */ + public void seal() { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot seal buffer in state " + stateName(state)); + } + state = STATE_SEALED; + } + + /** + * Rolls back a seal operation, transitioning from SEALED back to FILLING. + *

+ * Used when enqueue fails after a buffer has been sealed but before ownership + * was transferred to the I/O thread. + * + * @throws IllegalStateException if not in SEALED state + */ + public void rollbackSealForRetry() { + if (state != STATE_SEALED) { + throw new IllegalStateException("Cannot rollback seal in state " + stateName(state)); + } + state = STATE_FILLING; + } + + /** + * Marks the buffer as being sent, transitioning from SEALED to SENDING. + * Only the I/O thread should call this. + * + * @throws IllegalStateException if not in SEALED state + */ + public void markSending() { + if (state != STATE_SEALED) { + throw new IllegalStateException("Cannot mark sending in state " + stateName(state)); + } + state = STATE_SENDING; + } + + /** + * Marks the buffer as recycled, transitioning from SENDING to RECYCLED. + * This signals to the user thread that the buffer can be reused. + * Only the I/O thread should call this. + * + * @throws IllegalStateException if not in SENDING state + */ + public void markRecycled() { + if (state != STATE_SENDING) { + throw new IllegalStateException("Cannot mark recycled in state " + stateName(state)); + } + state = STATE_RECYCLED; + recycleLatch.countDown(); + } + + /** + * Waits for the buffer to be recycled (transition to RECYCLED state). + * Only the user thread should call this. + */ + public void awaitRecycled() { + try { + recycleLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** + * Waits for the buffer to be recycled with a timeout. + * + * @param timeout the maximum time to wait + * @param unit the time unit + * @return true if recycled, false if timeout elapsed + */ + public boolean awaitRecycled(long timeout, TimeUnit unit) { + try { + return recycleLatch.await(timeout, unit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** + * Resets the buffer to FILLING state, clearing all data. + * Only valid when in RECYCLED state or when the buffer is fresh. + * + * @throws IllegalStateException if in SEALED or SENDING state + */ + public void reset() { + int s = state; + if (s == STATE_SEALED || s == STATE_SENDING) { + throw new IllegalStateException("Cannot reset buffer in state " + stateName(s)); + } + bufferPos = 0; + rowCount = 0; + firstRowTimeNanos = 0; + maxSymbolId = -1; + batchId = nextBatchId++; + state = STATE_FILLING; + recycleLatch = new CountDownLatch(1); + } + + // ==================== LIFECYCLE ==================== + + @Override + public void close() { + if (bufferPtr != 0) { + Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS); + bufferPtr = 0; + bufferCapacity = 0; + } + } + + // ==================== UTILITIES ==================== + + /** + * Returns a human-readable name for the given state. + */ + public static String stateName(int state) { + switch (state) { + case STATE_FILLING: + return "FILLING"; + case STATE_SEALED: + return "SEALED"; + case STATE_SENDING: + return "SENDING"; + case STATE_RECYCLED: + return "RECYCLED"; + default: + return "UNKNOWN(" + state + ")"; + } + } + + @Override + public String toString() { + return "MicrobatchBuffer{" + + "batchId=" + batchId + + ", state=" + stateName(state) + + ", rows=" + rowCount + + ", bytes=" + bufferPos + + ", capacity=" + bufferCapacity + + '}'; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java new file mode 100644 index 0000000..5cab035 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java @@ -0,0 +1,289 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +/** + * A simple native memory buffer writer for encoding ILP v4 messages. + *

+ * This class provides write methods similar to HttpClient.Request but writes + * to a native memory buffer that can be sent over WebSocket. + *

+ * All multi-byte values are written in little-endian format unless otherwise specified. + */ +public class NativeBufferWriter implements IlpBufferWriter, QuietCloseable { + + private static final int DEFAULT_CAPACITY = 8192; + + private long bufferPtr; + private int capacity; + private int position; + + public NativeBufferWriter() { + this(DEFAULT_CAPACITY); + } + + public NativeBufferWriter(int initialCapacity) { + this.capacity = initialCapacity; + this.bufferPtr = Unsafe.malloc(capacity, MemoryTag.NATIVE_DEFAULT); + this.position = 0; + } + + /** + * Returns the buffer pointer. + */ + @Override + public long getBufferPtr() { + return bufferPtr; + } + + /** + * Returns the current write position (number of bytes written). + */ + @Override + public int getPosition() { + return position; + } + + /** + * Resets the buffer for reuse. + */ + @Override + public void reset() { + position = 0; + } + + /** + * Writes a single byte. + */ + @Override + public void putByte(byte value) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(bufferPtr + position, value); + position++; + } + + /** + * Writes a short (2 bytes, little-endian). + */ + @Override + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(bufferPtr + position, value); + position += 2; + } + + /** + * Writes an int (4 bytes, little-endian). + */ + @Override + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(bufferPtr + position, value); + position += 4; + } + + /** + * Writes a long (8 bytes, little-endian). + */ + @Override + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufferPtr + position, value); + position += 8; + } + + /** + * Writes a long in big-endian order. + */ + @Override + public void putLongBE(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufferPtr + position, Long.reverseBytes(value)); + position += 8; + } + + /** + * Writes a float (4 bytes, little-endian). + */ + @Override + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(bufferPtr + position, value); + position += 4; + } + + /** + * Writes a double (8 bytes, little-endian). + */ + @Override + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(bufferPtr + position, value); + position += 8; + } + + /** + * Writes a block of bytes from native memory. + */ + @Override + public void putBlockOfBytes(long from, long len) { + ensureCapacity((int) len); + Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len); + position += (int) len; + } + + /** + * Writes a varint (unsigned LEB128). + */ + @Override + public void putVarint(long value) { + while (value > 0x7F) { + putByte((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + putByte((byte) value); + } + + /** + * Writes a length-prefixed UTF-8 string. + */ + @Override + public void putString(String value) { + if (value == null || value.isEmpty()) { + putVarint(0); + return; + } + + int utf8Len = utf8Length(value); + putVarint(utf8Len); + putUtf8(value); + } + + /** + * Writes UTF-8 bytes directly without length prefix. + */ + @Override + public void putUtf8(String value) { + if (value == null || value.isEmpty()) { + return; + } + for (int i = 0, n = value.length(); i < n; i++) { + char c = value.charAt(i); + if (c < 0x80) { + putByte((byte) c); + } else if (c < 0x800) { + putByte((byte) (0xC0 | (c >> 6))); + putByte((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + char c2 = value.charAt(++i); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) (0xE0 | (c >> 12))); + putByte((byte) (0x80 | ((c >> 6) & 0x3F))); + putByte((byte) (0x80 | (c & 0x3F))); + } + } + } + + /** + * Returns the UTF-8 encoded length of a string. + */ + public static int utf8Length(String s) { + if (s == null) return 0; + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + i++; + len += 4; + } else { + len += 3; + } + } + return len; + } + + /** + * Patches an int value at the specified offset. + * Used for updating length fields after writing content. + */ + @Override + public void patchInt(int offset, int value) { + Unsafe.getUnsafe().putInt(bufferPtr + offset, value); + } + + /** + * Returns the current buffer capacity. + */ + @Override + public int getCapacity() { + return capacity; + } + + /** + * Skips the specified number of bytes, advancing the position. + * Used when data has been written directly to the buffer via getBufferPtr(). + * + * @param bytes number of bytes to skip + */ + @Override + public void skip(int bytes) { + position += bytes; + } + + /** + * Ensures the buffer has at least the specified additional capacity. + * + * @param needed additional bytes needed beyond current position + */ + @Override + public void ensureCapacity(int needed) { + if (position + needed > capacity) { + int newCapacity = Math.max(capacity * 2, position + needed); + bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT); + capacity = newCapacity; + } + } + + @Override + public void close() { + if (bufferPtr != 0) { + Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT); + bufferPtr = 0; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java new file mode 100644 index 0000000..dca05f2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java @@ -0,0 +1,247 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Reads server responses from WebSocket channel and updates InFlightWindow. + *

+ * This class runs a dedicated thread that: + *

    + *
  • Reads WebSocket frames from the server
    • + *
  • Parses binary responses containing ACK/error status
    • + *
  • Updates the InFlightWindow with acknowledgments or failures
    • + *
+ *

+ * Thread safety: This class is thread-safe. The reader thread processes + * responses independently of the sender thread. + */ +public class ResponseReader implements QuietCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); + + private static final int DEFAULT_READ_TIMEOUT_MS = 100; + private static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; + + private final WebSocketChannel channel; + private final InFlightWindow inFlightWindow; + private final Thread readerThread; + private final CountDownLatch shutdownLatch; + private final WebSocketResponse response; + + // Buffer for parsing responses + private final long parseBufferPtr; + private final int parseBufferSize; + + // State + private volatile boolean running; + private volatile Throwable lastError; + + // Statistics + private final AtomicLong totalAcks = new AtomicLong(0); + private final AtomicLong totalErrors = new AtomicLong(0); + + /** + * Creates a new response reader. + * + * @param channel the WebSocket channel to read from + * @param inFlightWindow the window to update with acknowledgments + */ + public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) { + if (channel == null) { + throw new IllegalArgumentException("channel cannot be null"); + } + if (inFlightWindow == null) { + throw new IllegalArgumentException("inFlightWindow cannot be null"); + } + + this.channel = channel; + this.inFlightWindow = inFlightWindow; + this.response = new WebSocketResponse(); + + // Allocate parse buffer (enough for max response) + this.parseBufferSize = 2048; + this.parseBufferPtr = Unsafe.malloc(parseBufferSize, MemoryTag.NATIVE_DEFAULT); + + this.running = true; + this.shutdownLatch = new CountDownLatch(1); + + // Start reader thread + this.readerThread = new Thread(this::readLoop, "questdb-websocket-response-reader"); + this.readerThread.setDaemon(true); + this.readerThread.start(); + + LOG.info("Response reader started"); + } + + /** + * Returns the last error that occurred, or null if no error. + */ + public Throwable getLastError() { + return lastError; + } + + /** + * Returns true if the reader is still running. + */ + public boolean isRunning() { + return running; + } + + /** + * Returns total successful acknowledgments received. + */ + public long getTotalAcks() { + return totalAcks.get(); + } + + /** + * Returns total error responses received. + */ + public long getTotalErrors() { + return totalErrors.get(); + } + + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing response reader"); + + running = false; + + // Wait for reader thread to finish + try { + shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Free parse buffer + if (parseBufferPtr != 0) { + Unsafe.free(parseBufferPtr, parseBufferSize, MemoryTag.NATIVE_DEFAULT); + } + + LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + } + + // ==================== Reader Thread ==================== + + /** + * Main read loop that processes incoming WebSocket frames. + */ + private void readLoop() { + LOG.info("Read loop started"); + + try { + while (running && channel.isConnected()) { + try { + // Non-blocking read with short timeout + boolean received = channel.receiveFrame(new ResponseHandlerImpl(), DEFAULT_READ_TIMEOUT_MS); + if (!received) { + // No frame available, continue polling + continue; + } + } catch (LineSenderException e) { + if (running) { + LOG.error("Error reading response: {}", e.getMessage()); + lastError = e; + } + // Continue trying to read unless we're shutting down + } catch (Throwable t) { + if (running) { + LOG.error("Unexpected error in read loop: {}", String.valueOf(t)); + lastError = t; + } + break; + } + } + } finally { + shutdownLatch.countDown(); + LOG.info("Read loop stopped"); + } + } + + /** + * Handler for received WebSocket frames. + */ + private class ResponseHandlerImpl implements WebSocketChannel.ResponseHandler { + + @Override + public void onBinaryMessage(long payload, int length) { + if (length < WebSocketResponse.MIN_RESPONSE_SIZE) { + LOG.error("Response too short [length={}]", length); + return; + } + + // Parse response from binary payload + if (!response.readFrom(payload, length)) { + LOG.error("Failed to parse response"); + return; + } + + long sequence = response.getSequence(); + + if (response.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + int acked = inFlightWindow.acknowledgeUpTo(sequence); + if (acked > 0) { + totalAcks.addAndGet(acked); + LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); + } else { + LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); + } + } else { + // Error - fail the batch + String errorMessage = response.getErrorMessage(); + LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage); + + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + response.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + totalErrors.incrementAndGet(); + } + } + + @Override + public void onClose(int code, String reason) { + LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason); + running = false; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java new file mode 100644 index 0000000..f5be4a4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java @@ -0,0 +1,668 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; + +import javax.net.SocketFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Base64; + +/** + * WebSocket client channel for ILP v4 binary streaming. + *

+ * This class handles: + *

    + *
  • HTTP upgrade handshake to establish WebSocket connection
    • + *
  • Binary frame encoding with client-side masking (RFC 6455)
    • + *
  • Ping/pong for connection keepalive
    • + *
  • Close handshake
    • + *
+ *

+ * Thread safety: This class is NOT thread-safe. It should only be accessed + * from a single thread at a time. + */ +public class WebSocketChannel implements QuietCloseable { + + private static final int DEFAULT_BUFFER_SIZE = 65536; + private static final int MAX_FRAME_HEADER_SIZE = 14; // 2 + 8 + 4 (header + extended len + mask) + + // Connection state + private final String host; + private final int port; + private final String path; + private final boolean tlsEnabled; + private final boolean tlsValidationEnabled; + + // Socket I/O + private Socket socket; + private InputStream in; + private OutputStream out; + + // Pre-allocated send buffer (native memory) + private long sendBufferPtr; + private int sendBufferSize; + + // Pre-allocated receive buffer (native memory) + private long recvBufferPtr; + private int recvBufferSize; + private int recvBufferPos; // Write position + private int recvBufferReadPos; // Read position + + // Frame parser (reused) + private final WebSocketFrameParser frameParser; + + // Random for mask key generation + private final Rnd rnd; + + // Timeouts + private int connectTimeoutMs = 10_000; + private int readTimeoutMs = 30_000; + + // State + private boolean connected; + private boolean closed; + + // Temporary byte array for handshake (allocated once) + private final byte[] handshakeBuffer = new byte[4096]; + + public WebSocketChannel(String url, boolean tlsEnabled) { + this(url, tlsEnabled, true); + } + + public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEnabled) { + // Parse URL: ws://host:port/path or wss://host:port/path + String remaining = url; + if (remaining.startsWith("wss://")) { + remaining = remaining.substring(6); + this.tlsEnabled = true; + } else if (remaining.startsWith("ws://")) { + remaining = remaining.substring(5); + this.tlsEnabled = tlsEnabled; + } else { + this.tlsEnabled = tlsEnabled; + } + + int slashIdx = remaining.indexOf('/'); + String hostPort; + if (slashIdx >= 0) { + hostPort = remaining.substring(0, slashIdx); + this.path = remaining.substring(slashIdx); + } else { + hostPort = remaining; + this.path = "/"; + } + + int colonIdx = hostPort.lastIndexOf(':'); + if (colonIdx >= 0) { + this.host = hostPort.substring(0, colonIdx); + this.port = Integer.parseInt(hostPort.substring(colonIdx + 1)); + } else { + this.host = hostPort; + this.port = this.tlsEnabled ? 443 : 80; + } + + this.tlsValidationEnabled = tlsValidationEnabled; + this.frameParser = new WebSocketFrameParser(); + this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + + // Allocate native buffers + this.sendBufferSize = DEFAULT_BUFFER_SIZE; + this.sendBufferPtr = Unsafe.malloc(sendBufferSize, MemoryTag.NATIVE_DEFAULT); + + this.recvBufferSize = DEFAULT_BUFFER_SIZE; + this.recvBufferPtr = Unsafe.malloc(recvBufferSize, MemoryTag.NATIVE_DEFAULT); + this.recvBufferPos = 0; + this.recvBufferReadPos = 0; + + this.connected = false; + this.closed = false; + } + + /** + * Sets the connection timeout. + */ + public WebSocketChannel setConnectTimeout(int timeoutMs) { + this.connectTimeoutMs = timeoutMs; + return this; + } + + /** + * Sets the read timeout. + */ + public WebSocketChannel setReadTimeout(int timeoutMs) { + this.readTimeoutMs = timeoutMs; + return this; + } + + /** + * Connects to the WebSocket server. + * Performs TCP connection and HTTP upgrade handshake. + */ + public void connect() { + if (connected) { + return; + } + if (closed) { + throw new LineSenderException("WebSocket channel is closed"); + } + + try { + // Create socket + SocketFactory socketFactory = tlsEnabled ? createSslSocketFactory() : SocketFactory.getDefault(); + socket = socketFactory.createSocket(); + socket.connect(new java.net.InetSocketAddress(host, port), connectTimeoutMs); + socket.setSoTimeout(readTimeoutMs); + socket.setTcpNoDelay(true); + + in = socket.getInputStream(); + out = socket.getOutputStream(); + + // Perform WebSocket handshake + performHandshake(); + + connected = true; + } catch (IOException e) { + closeQuietly(); + throw new LineSenderException("Failed to connect to WebSocket server: " + e.getMessage(), e); + } + } + + /** + * Sends binary data as a WebSocket binary frame. + * The data is read from native memory at the given pointer. + * + * @param dataPtr pointer to the data + * @param length length of data in bytes + */ + public void sendBinary(long dataPtr, int length) { + ensureConnected(); + sendFrame(WebSocketOpcode.BINARY, dataPtr, length); + } + + /** + * Sends a ping frame. + */ + public void sendPing() { + ensureConnected(); + sendFrame(WebSocketOpcode.PING, 0, 0); + } + + /** + * Receives and processes incoming frames. + * Handles ping/pong automatically. + * + * @param handler callback for received binary messages (may be null) + * @param timeoutMs read timeout in milliseconds + * @return true if a frame was received, false on timeout + */ + public boolean receiveFrame(ResponseHandler handler, int timeoutMs) { + ensureConnected(); + try { + int oldTimeout = socket.getSoTimeout(); + socket.setSoTimeout(timeoutMs); + try { + return doReceiveFrame(handler); + } finally { + socket.setSoTimeout(oldTimeout); + } + } catch (SocketTimeoutException e) { + return false; + } catch (IOException e) { + throw new LineSenderException("Failed to receive WebSocket frame: " + e.getMessage(), e); + } + } + + /** + * Sends a close frame and closes the connection. + */ + @Override + public void close() { + if (closed) { + return; + } + closed = true; + + try { + if (connected) { + // Send close frame + sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); + } + } catch (Exception e) { + // Ignore errors during close + } + + closeQuietly(); + + // Free native memory + if (sendBufferPtr != 0) { + Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); + sendBufferPtr = 0; + } + if (recvBufferPtr != 0) { + Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); + recvBufferPtr = 0; + } + } + + public boolean isConnected() { + return connected && !closed; + } + + // ==================== Private methods ==================== + + private void ensureConnected() { + if (closed) { + throw new LineSenderException("WebSocket channel is closed"); + } + if (!connected) { + throw new LineSenderException("WebSocket channel is not connected"); + } + } + + private SocketFactory createSslSocketFactory() { + try { + if (!tlsValidationEnabled) { + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new TrustManager[]{new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] certs, String t) {} + public void checkServerTrusted(X509Certificate[] certs, String t) {} + public X509Certificate[] getAcceptedIssuers() { return null; } + }}, new SecureRandom()); + return sslContext.getSocketFactory(); + } + return SSLSocketFactory.getDefault(); + } catch (Exception e) { + throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); + } + } + + private void performHandshake() throws IOException { + // Generate random key (16 bytes, base64 encoded = 24 chars) + byte[] keyBytes = new byte[16]; + for (int i = 0; i < 16; i++) { + keyBytes[i] = (byte) rnd.nextInt(256); + } + String key = Base64.getEncoder().encodeToString(keyBytes); + + // Build HTTP upgrade request + StringBuilder request = new StringBuilder(); + request.append("GET ").append(path).append(" HTTP/1.1\r\n"); + request.append("Host: ").append(host); + if ((tlsEnabled && port != 443) || (!tlsEnabled && port != 80)) { + request.append(":").append(port); + } + request.append("\r\n"); + request.append("Upgrade: websocket\r\n"); + request.append("Connection: Upgrade\r\n"); + request.append("Sec-WebSocket-Key: ").append(key).append("\r\n"); + request.append("Sec-WebSocket-Version: 13\r\n"); + request.append("\r\n"); + + // Send request + byte[] requestBytes = request.toString().getBytes(StandardCharsets.US_ASCII); + out.write(requestBytes); + out.flush(); + + // Read response + int responseLen = readHttpResponse(); + + // Parse response + String response = new String(handshakeBuffer, 0, responseLen, StandardCharsets.US_ASCII); + + // Check status line + if (!response.startsWith("HTTP/1.1 101")) { + throw new IOException("WebSocket handshake failed: " + response.split("\r\n")[0]); + } + + // Verify Sec-WebSocket-Accept + String expectedAccept = WebSocketHandshake.computeAcceptKey(key); + if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) { + throw new IOException("Invalid Sec-WebSocket-Accept in handshake response"); + } + } + + private int readHttpResponse() throws IOException { + int pos = 0; + int consecutiveCrLf = 0; + + while (pos < handshakeBuffer.length) { + int b = in.read(); + if (b < 0) { + throw new IOException("Connection closed during handshake"); + } + handshakeBuffer[pos++] = (byte) b; + + // Look for \r\n\r\n + if (b == '\r' || b == '\n') { + if ((consecutiveCrLf == 0 && b == '\r') || + (consecutiveCrLf == 1 && b == '\n') || + (consecutiveCrLf == 2 && b == '\r') || + (consecutiveCrLf == 3 && b == '\n')) { + consecutiveCrLf++; + if (consecutiveCrLf == 4) { + return pos; + } + } else { + consecutiveCrLf = (b == '\r') ? 1 : 0; + } + } else { + consecutiveCrLf = 0; + } + } + throw new IOException("HTTP response too large"); + } + + private void sendFrame(int opcode, long payloadPtr, int payloadLen) { + // Generate mask key + int maskKey = rnd.nextInt(); + + // Calculate required buffer size + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int frameSize = headerSize + payloadLen; + + // Ensure buffer is large enough + ensureSendBufferSize(frameSize); + + // Write frame header with mask + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, opcode, payloadLen, maskKey); + + // Copy payload to buffer after header + if (payloadLen > 0) { + Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); + // Mask the payload in place + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); + } + + // Send frame + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); + } + } + + private void sendCloseFrame(int code, String reason) { + int maskKey = rnd.nextInt(); + + // Close payload: 2-byte code + optional reason + int reasonLen = (reason != null) ? reason.length() : 0; + int payloadLen = 2 + reasonLen; + + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int frameSize = headerSize + payloadLen; + + ensureSendBufferSize(frameSize); + + // Write header + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, WebSocketOpcode.CLOSE, payloadLen, maskKey); + + // Write close code (big-endian) + long payloadStart = sendBufferPtr + headerWritten; + Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF)); + + // Write reason if present + if (reason != null) { + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + for (int i = 0; i < reasonBytes.length; i++) { + Unsafe.getUnsafe().putByte(payloadStart + 2 + i, reasonBytes[i]); + } + } + + // Mask payload + WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey); + + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + // Ignore errors during close + } + } + + private boolean doReceiveFrame(ResponseHandler handler) throws IOException { + // First, try to parse any data already in the buffer + // This handles the case where multiple frames arrived in a single TCP read + if (recvBufferPos > recvBufferReadPos) { + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + // result == null means we need more data, continue to read + } + + // Read more data into receive buffer + int bytesRead = readFromSocket(); + if (bytesRead <= 0) { + return false; + } + + // Try parsing again with the new data + Boolean result = tryParseFrame(handler); + return result != null && result; + } + + /** + * Tries to parse a frame from the receive buffer. + * @return true if frame processed, false if error, null if need more data + */ + private Boolean tryParseFrame(ResponseHandler handler) throws IOException { + frameParser.reset(); + int consumed = frameParser.parse( + recvBufferPtr + recvBufferReadPos, + recvBufferPtr + recvBufferPos); + + if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE) { + return null; // Need more data + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) { + throw new IOException("WebSocket frame parse error: " + frameParser.getErrorCode()); + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { + long payloadPtr = recvBufferPtr + recvBufferReadPos + frameParser.getHeaderSize(); + int payloadLen = (int) frameParser.getPayloadLength(); + + // Handle control frames + int opcode = frameParser.getOpcode(); + switch (opcode) { + case WebSocketOpcode.PING: + sendPongFrame(payloadPtr, payloadLen); + break; + case WebSocketOpcode.PONG: + // Ignore pong + break; + case WebSocketOpcode.CLOSE: + connected = false; + if (handler != null) { + int closeCode = 0; + if (payloadLen >= 2) { + closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8) + | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF); + } + handler.onClose(closeCode, null); + } + break; + case WebSocketOpcode.BINARY: + if (handler != null) { + handler.onBinaryMessage(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.TEXT: + // Ignore text frames for now + break; + } + + // Advance read position + recvBufferReadPos += consumed; + + // Compact buffer if needed + if (recvBufferReadPos > 0) { + int remaining = recvBufferPos - recvBufferReadPos; + if (remaining > 0) { + Unsafe.getUnsafe().copyMemory( + recvBufferPtr + recvBufferReadPos, + recvBufferPtr, + remaining); + } + recvBufferPos = remaining; + recvBufferReadPos = 0; + } + + return true; + } + + return false; + } + + private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { + int maskKey = rnd.nextInt(); + int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); + int frameSize = headerSize + pingPayloadLen; + + ensureSendBufferSize(frameSize); + + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); + + if (pingPayloadLen > 0) { + Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); + } + + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + // Ignore pong send errors + } + } + + private void ensureSendBufferSize(int required) { + if (required > sendBufferSize) { + int newSize = Math.max(required, sendBufferSize * 2); + sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + sendBufferSize = newSize; + } + } + + private void writeToSocket(long ptr, int len) throws IOException { + // Copy to temp array for socket write (unavoidable with OutputStream) + // Use separate write buffer to avoid race with read thread + byte[] temp = getWriteTempBuffer(len); + for (int i = 0; i < len; i++) { + temp[i] = Unsafe.getUnsafe().getByte(ptr + i); + } + out.write(temp, 0, len); + out.flush(); + } + + private int readFromSocket() throws IOException { + // Ensure space in receive buffer + int available = recvBufferSize - recvBufferPos; + if (available < 1024) { + // Grow buffer + int newSize = recvBufferSize * 2; + recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + recvBufferSize = newSize; + available = recvBufferSize - recvBufferPos; + } + + // Read into temp array then copy to native buffer + // Use separate read buffer to avoid race with write thread + byte[] temp = getReadTempBuffer(available); + int bytesRead = in.read(temp, 0, available); + if (bytesRead > 0) { + for (int i = 0; i < bytesRead; i++) { + Unsafe.getUnsafe().putByte(recvBufferPtr + recvBufferPos + i, temp[i]); + } + recvBufferPos += bytesRead; + } + return bytesRead; + } + + // Separate temp buffers for read and write to avoid race conditions + // between send queue thread and response reader thread + private byte[] writeTempBuffer; + private byte[] readTempBuffer; + + private byte[] getWriteTempBuffer(int minSize) { + if (writeTempBuffer == null || writeTempBuffer.length < minSize) { + writeTempBuffer = new byte[Math.max(minSize, 8192)]; + } + return writeTempBuffer; + } + + private byte[] getReadTempBuffer(int minSize) { + if (readTempBuffer == null || readTempBuffer.length < minSize) { + readTempBuffer = new byte[Math.max(minSize, 8192)]; + } + return readTempBuffer; + } + + private void closeQuietly() { + connected = false; + if (socket != null) { + try { + socket.close(); + } catch (IOException e) { + // Ignore + } + socket = null; + } + in = null; + out = null; + } + + /** + * Callback interface for received WebSocket messages. + */ + public interface ResponseHandler { + void onBinaryMessage(long payload, int length); + void onClose(int code, String reason); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java new file mode 100644 index 0000000..42e74af --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java @@ -0,0 +1,283 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; + +/** + * Binary response format for WebSocket ILP v4 protocol. + *

+ * Response format (little-endian): + * 

+ * +--------+----------+------------------+
+ * | status | sequence | error (if any)   |
+ * | 1 byte | 8 bytes  | 2 bytes + UTF-8  |
+ * +--------+----------+------------------+
+ *
+ *

+ * Status codes: + *

    + *
  • 0: Success (ACK)
    • + *
  • 1: Parse error
    • + *
  • 2: Schema error
    • + *
  • 3: Write error
    • + *
  • 4: Security error
    • + *
  • 255: Internal error
    • + *
+ *

+ * The sequence number allows correlation with the original request. + * Error message is only present when status != 0. + */ +public class WebSocketResponse { + + // Status codes + public static final byte STATUS_OK = 0; + public static final byte STATUS_PARSE_ERROR = 1; + public static final byte STATUS_SCHEMA_ERROR = 2; + public static final byte STATUS_WRITE_ERROR = 3; + public static final byte STATUS_SECURITY_ERROR = 4; + public static final byte STATUS_INTERNAL_ERROR = (byte) 255; + + // Minimum response size: status (1) + sequence (8) + public static final int MIN_RESPONSE_SIZE = 9; + public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length + public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; + + private byte status; + private long sequence; + private String errorMessage; + + public WebSocketResponse() { + this.status = STATUS_OK; + this.sequence = 0; + this.errorMessage = null; + } + + /** + * Creates a success response. + */ + public static WebSocketResponse success(long sequence) { + WebSocketResponse response = new WebSocketResponse(); + response.status = STATUS_OK; + response.sequence = sequence; + return response; + } + + /** + * Creates an error response. + */ + public static WebSocketResponse error(long sequence, byte status, String errorMessage) { + WebSocketResponse response = new WebSocketResponse(); + response.status = status; + response.sequence = sequence; + response.errorMessage = errorMessage; + return response; + } + + /** + * Validates binary response framing without allocating. + *

+ * Accepted formats: + *

    + *
  • OK: exactly 9 bytes (status + sequence)
    • + *
  • Error: exactly 11 + errorLength bytes
    • + *
+ * + * @param ptr response buffer pointer + * @param length response frame payload length + * @return true if payload structure is valid + */ + public static boolean isStructurallyValid(long ptr, int length) { + if (length < MIN_RESPONSE_SIZE) { + return false; + } + + byte status = Unsafe.getUnsafe().getByte(ptr); + if (status == STATUS_OK) { + return length == MIN_RESPONSE_SIZE; + } + + if (length < MIN_ERROR_RESPONSE_SIZE) { + return false; + } + + int msgLen = Unsafe.getUnsafe().getShort(ptr + MIN_RESPONSE_SIZE) & 0xFFFF; + return length == MIN_ERROR_RESPONSE_SIZE + msgLen; + } + + /** + * Returns true if this is a success response. + */ + public boolean isSuccess() { + return status == STATUS_OK; + } + + /** + * Returns the status code. + */ + public byte getStatus() { + return status; + } + + /** + * Returns the sequence number. + */ + public long getSequence() { + return sequence; + } + + /** + * Returns the error message, or null for success responses. + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * Returns a human-readable status name. + */ + public String getStatusName() { + switch (status) { + case STATUS_OK: + return "OK"; + case STATUS_PARSE_ERROR: + return "PARSE_ERROR"; + case STATUS_SCHEMA_ERROR: + return "SCHEMA_ERROR"; + case STATUS_WRITE_ERROR: + return "WRITE_ERROR"; + case STATUS_SECURITY_ERROR: + return "SECURITY_ERROR"; + case STATUS_INTERNAL_ERROR: + return "INTERNAL_ERROR"; + default: + return "UNKNOWN(" + (status & 0xFF) + ")"; + } + } + + /** + * Calculates the serialized size of this response. + */ + public int serializedSize() { + int size = MIN_RESPONSE_SIZE; + if (errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + size += 2 + msgLen; // 2 bytes for length prefix + } + return size; + } + + /** + * Writes this response to native memory. + * + * @param ptr destination address + * @return number of bytes written + */ + public int writeTo(long ptr) { + int offset = 0; + + // Status (1 byte) + Unsafe.getUnsafe().putByte(ptr + offset, status); + offset += 1; + + // Sequence (8 bytes, little-endian) + Unsafe.getUnsafe().putLong(ptr + offset, sequence); + offset += 8; + + // Error message (if any) + if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + + // Length prefix (2 bytes, little-endian) + Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen); + offset += 2; + + // Message bytes + for (int i = 0; i < msgLen; i++) { + Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]); + } + offset += msgLen; + } + + return offset; + } + + /** + * Reads a response from native memory. + * + * @param ptr source address + * @param length available bytes + * @return true if successfully parsed, false if not enough data + */ + public boolean readFrom(long ptr, int length) { + if (length < MIN_RESPONSE_SIZE) { + return false; + } + + int offset = 0; + + // Status (1 byte) + status = Unsafe.getUnsafe().getByte(ptr + offset); + offset += 1; + + // Sequence (8 bytes, little-endian) + sequence = Unsafe.getUnsafe().getLong(ptr + offset); + offset += 8; + + // Error message (if status != OK and more data available) + if (status != STATUS_OK && length > offset + 2) { + int msgLen = Unsafe.getUnsafe().getShort(ptr + offset) & 0xFFFF; + offset += 2; + + if (length >= offset + msgLen && msgLen > 0) { + byte[] msgBytes = new byte[msgLen]; + for (int i = 0; i < msgLen; i++) { + msgBytes[i] = Unsafe.getUnsafe().getByte(ptr + offset + i); + } + errorMessage = new String(msgBytes, StandardCharsets.UTF_8); + offset += msgLen; + } + } else { + errorMessage = null; + } + + return true; + } + + @Override + public String toString() { + if (isSuccess()) { + return "WebSocketResponse{status=OK, seq=" + sequence + "}"; + } else { + return "WebSocketResponse{status=" + getStatusName() + ", seq=" + sequence + + ", error=" + errorMessage + "}"; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java new file mode 100644 index 0000000..b34926e --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java @@ -0,0 +1,693 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.Nullable; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Asynchronous I/O handler for WebSocket microbatch transmission. + *

+ * This class manages a dedicated I/O thread that handles both: + *

    + *
  • Sending batches from a bounded queue
    • + *
  • Receiving and processing server ACK responses
    • + *
+ * Using a single thread eliminates concurrency issues with the WebSocket channel. + *

+ * Thread safety: + *

    + *
  • The send queue is thread-safe for concurrent access
    • + *
  • Only the I/O thread interacts with the WebSocket channel
    • + *
  • Buffer state transitions ensure safe hand-over
    • + *
+ *

+ * Backpressure: + *

    + *
  • When the queue is full, {@link #enqueue} blocks
    • + *
  • This propagates backpressure to the user thread
    • + *
+ */ +public class WebSocketSendQueue implements QuietCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); + + // Default configuration + public static final int DEFAULT_QUEUE_CAPACITY = 16; + public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; + public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; + + // Single pending buffer slot (double-buffering means at most 1 item in queue) + // Zero allocation - just a volatile reference handoff + private volatile MicrobatchBuffer pendingBuffer; + + // The WebSocket client for I/O (single-threaded access only) + private final WebSocketClient client; + + // Optional InFlightWindow for tracking sent batches awaiting ACK + @Nullable + private final InFlightWindow inFlightWindow; + + // The I/O thread for async send/receive + private final Thread ioThread; + + // Running state + private volatile boolean running; + private volatile boolean shuttingDown; + + // Synchronization for flush/close + private final CountDownLatch shutdownLatch; + + // Error handling + private volatile Throwable lastError; + + // Statistics - sending + private final AtomicLong totalBatchesSent = new AtomicLong(0); + private final AtomicLong totalBytesSent = new AtomicLong(0); + + // Statistics - receiving + private final AtomicLong totalAcks = new AtomicLong(0); + private final AtomicLong totalErrors = new AtomicLong(0); + + // Counter for batches currently being processed by the I/O thread + // This tracks batches that have been dequeued but not yet fully sent + private final AtomicInteger processingCount = new AtomicInteger(0); + + // Lock for all coordination between user thread and I/O thread. + // Used for: queue poll + processingCount increment atomicity, + // flush() waiting, I/O thread waiting when idle. + private final Object processingLock = new Object(); + + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + + // Response parsing + private final WebSocketResponse response = new WebSocketResponse(); + private final ResponseHandler responseHandler = new ResponseHandler(); + + // Configuration + private final long enqueueTimeoutMs; + private final long shutdownTimeoutMs; + + // ==================== Pending Buffer Operations (zero allocation) ==================== + + private boolean offerPending(MicrobatchBuffer buffer) { + if (pendingBuffer != null) { + return false; // slot occupied + } + pendingBuffer = buffer; + return true; + } + + private MicrobatchBuffer pollPending() { + MicrobatchBuffer buffer = pendingBuffer; + if (buffer != null) { + pendingBuffer = null; + } + return buffer; + } + + private boolean isPendingEmpty() { + return pendingBuffer == null; + } + + private int getPendingSize() { + return pendingBuffer == null ? 0 : 1; + } + + /** + * Creates a new send queue with default configuration. + * + * @param client the WebSocket client for I/O + */ + public WebSocketSendQueue(WebSocketClient client) { + this(client, null, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + + /** + * Creates a new send queue with InFlightWindow for tracking sent batches. + * + * @param client the WebSocket client for I/O + * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) + */ + public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow) { + this(client, inFlightWindow, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + + /** + * Creates a new send queue with custom configuration. + * + * @param client the WebSocket client for I/O + * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) + * @param queueCapacity maximum number of pending batches + * @param enqueueTimeoutMs timeout for enqueue operations (ms) + * @param shutdownTimeoutMs timeout for graceful shutdown (ms) + */ + public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow, + int queueCapacity, long enqueueTimeoutMs, long shutdownTimeoutMs) { + if (client == null) { + throw new IllegalArgumentException("client cannot be null"); + } + if (queueCapacity <= 0) { + throw new IllegalArgumentException("queueCapacity must be positive"); + } + + this.client = client; + this.inFlightWindow = inFlightWindow; + this.enqueueTimeoutMs = enqueueTimeoutMs; + this.shutdownTimeoutMs = shutdownTimeoutMs; + this.running = true; + this.shuttingDown = false; + this.shutdownLatch = new CountDownLatch(1); + + // Start the I/O thread (handles both sending and receiving) + this.ioThread = new Thread(this::ioLoop, "questdb-websocket-io"); + this.ioThread.setDaemon(true); + this.ioThread.start(); + + LOG.info("WebSocket I/O thread started [capacity={}]", queueCapacity); + } + + /** + * Enqueues a sealed buffer for sending. + *

+ * The buffer must be in SEALED state. After this method returns successfully, + * ownership of the buffer transfers to the send queue. + * + * @param buffer the sealed buffer to send + * @return true if enqueued successfully + * @throws LineSenderException if the buffer is not sealed or an error occurred + */ + public boolean enqueue(MicrobatchBuffer buffer) { + if (buffer == null) { + throw new IllegalArgumentException("buffer cannot be null"); + } + if (!buffer.isSealed()) { + throw new LineSenderException("Buffer must be sealed before enqueue, state=" + + MicrobatchBuffer.stateName(buffer.getState())); + } + if (!running || shuttingDown) { + throw new LineSenderException("Send queue is not running"); + } + + // Check for errors from I/O thread + checkError(); + + final long deadline = System.currentTimeMillis() + enqueueTimeoutMs; + synchronized (processingLock) { + while (true) { + if (!running || shuttingDown) { + throw new LineSenderException("Send queue is not running"); + } + checkError(); + + if (offerPending(buffer)) { + processingLock.notifyAll(); + break; + } + + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Enqueue timeout after " + enqueueTimeoutMs + "ms"); + } + try { + processingLock.wait(Math.min(10, remaining)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new LineSenderException("Interrupted while enqueueing", e); + } + } + } + + LOG.debug("Enqueued batch [id={}, bytes={}, rows={}]", buffer.getBatchId(), buffer.getBufferPos(), buffer.getRowCount()); + return true; + } + + /** + * Waits for all pending batches to be sent. + *

+ * This method blocks until the queue is empty and all in-flight sends complete. + * It does not close the queue - new batches can still be enqueued after flush. + * + * @throws LineSenderException if an error occurs during flush + */ + public void flush() { + checkError(); + + long startTime = System.currentTimeMillis(); + + // Wait under lock - I/O thread will notify when processingCount decrements + synchronized (processingLock) { + while (running) { + // Atomically check: queue empty AND not processing + if (isPendingEmpty() && processingCount.get() == 0) { + break; // All done + } + + try { + processingLock.wait(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new LineSenderException("Interrupted while flushing", e); + } + + // Check timeout + if (System.currentTimeMillis() - startTime > enqueueTimeoutMs) { + throw new LineSenderException("Flush timeout after " + enqueueTimeoutMs + "ms, " + + "queue=" + getPendingSize() + ", processing=" + processingCount.get()); + } + + // Check for errors + checkError(); + } + } + + // If loop exited because running=false we still need to surface the root cause. + checkError(); + + LOG.debug("Flush complete"); + } + + /** + * Returns the number of batches waiting to be sent. + */ + public int getPendingCount() { + return getPendingSize(); + } + + /** + * Returns true if the queue is empty. + */ + public boolean isEmpty() { + return isPendingEmpty(); + } + + /** + * Returns true if the queue is still running. + */ + public boolean isRunning() { + return running && !shuttingDown; + } + + /** + * Returns the total number of batches sent. + */ + public long getTotalBatchesSent() { + return totalBatchesSent.get(); + } + + /** + * Returns the total number of bytes sent. + */ + public long getTotalBytesSent() { + return totalBytesSent.get(); + } + + /** + * Returns the last error that occurred in the I/O thread, or null if no error. + */ + public Throwable getLastError() { + return lastError; + } + + /** + * Closes the send queue gracefully. + *

+ * This method: + * 1. Stops accepting new batches + * 2. Waits for pending batches to be sent + * 3. Stops the I/O thread + *

+ * Note: This does NOT close the WebSocket channel - that's the caller's responsibility. + */ + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize()); + + // Signal shutdown + shuttingDown = true; + + // Wait for pending batches to be sent + long startTime = System.currentTimeMillis(); + while (!isPendingEmpty()) { + if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { + LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); + break; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + // Stop the I/O thread + running = false; + + // Wake up I/O thread if it's blocked on processingLock.wait() + synchronized (processingLock) { + processingLock.notifyAll(); + } + ioThread.interrupt(); + + // Wait for I/O thread to finish + try { + shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get()); + } + + // ==================== I/O Thread ==================== + + /** + * I/O loop states for the state machine. + *

    + *
  • IDLE: queue empty, no in-flight batches - can block waiting for work
    • + *
  • ACTIVE: have batches to send - non-blocking loop
    • + *
  • DRAINING: queue empty but ACKs pending - poll for ACKs, short wait
    • + *
+ */ + private enum IoState { + IDLE, ACTIVE, DRAINING + } + + /** + * The main I/O loop that handles both sending batches and receiving ACKs. + *

+ * Uses a state machine: + *

    + *
  • IDLE: block on processingLock.wait() until work arrives
    • + *
  • ACTIVE: non-blocking poll queue, send batches, check for ACKs
    • + *
  • DRAINING: no batches but ACKs pending - poll for ACKs with short wait
    • + *
+ */ + private void ioLoop() { + LOG.info("I/O loop started"); + + try { + while (running || !isPendingEmpty()) { + MicrobatchBuffer batch = null; + boolean hasInFlight = (inFlightWindow != null && inFlightWindow.getInFlightCount() > 0); + IoState state = computeState(hasInFlight); + + switch (state) { + case IDLE: + // Nothing to do - wait for work under lock + synchronized (processingLock) { + // Re-check under lock to avoid missed wakeup + if (isPendingEmpty() && running) { + try { + processingLock.wait(100); + } catch (InterruptedException e) { + if (!running) return; + } + } + } + break; + + case ACTIVE: + case DRAINING: + // Try to receive any pending ACKs (non-blocking) + if (hasInFlight && client.isConnected()) { + tryReceiveAcks(); + } + + // Try to dequeue and send a batch + boolean hasWindowSpace = (inFlightWindow == null || inFlightWindow.hasWindowSpace()); + if (hasWindowSpace) { + // Atomically: poll queue + increment processingCount + synchronized (processingLock) { + batch = pollPending(); + if (batch != null) { + processingCount.incrementAndGet(); + } + } + + if (batch != null) { + try { + safeSendBatch(batch); + } finally { + // Atomically: decrement + notify flush() + synchronized (processingLock) { + processingCount.decrementAndGet(); + processingLock.notifyAll(); + } + } + } + } + + // In DRAINING state with no work, short wait to avoid busy loop + if (state == IoState.DRAINING && batch == null) { + synchronized (processingLock) { + try { + processingLock.wait(10); + } catch (InterruptedException e) { + if (!running) return; + } + } + } + break; + } + } + } finally { + shutdownLatch.countDown(); + LOG.info("I/O loop stopped [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + } + } + + /** + * Computes the current I/O state based on queue and in-flight status. + */ + private IoState computeState(boolean hasInFlight) { + if (!isPendingEmpty()) { + return IoState.ACTIVE; + } else if (hasInFlight) { + return IoState.DRAINING; + } else { + return IoState.IDLE; + } + } + + /** + * Tries to receive ACKs from the server (non-blocking). + */ + private void tryReceiveAcks() { + try { + client.tryReceiveFrame(responseHandler); + } catch (Exception e) { + if (running) { + LOG.error("Error receiving response: {}", e.getMessage()); + failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); + } + } + } + + /** + * Sends a batch with error handling. Does NOT manage processingCount. + */ + private void safeSendBatch(MicrobatchBuffer batch) { + try { + sendBatch(batch); + } catch (Throwable t) { + LOG.error("Error sending batch [id={}]{}", batch.getBatchId(), "", t); + failTransport(new LineSenderException("Error sending batch " + batch.getBatchId() + ": " + t.getMessage(), t)); + // Mark as recycled even on error to allow cleanup + if (batch.isSealed()) { + batch.markSending(); + } + if (batch.isSending()) { + batch.markRecycled(); + } + } + } + + /** + * Sends a single batch over the WebSocket channel. + */ + private void sendBatch(MicrobatchBuffer batch) { + // Transition state: SEALED -> SENDING + batch.markSending(); + + // Use our own sequence counter (must match server's messageSequence) + long batchSequence = nextBatchSequence++; + int bytes = batch.getBufferPos(); + int rows = batch.getRowCount(); + + LOG.debug("Sending batch [seq={}, bytes={}, rows={}, bufferId={}]", batchSequence, bytes, rows, batch.getBatchId()); + + // Add to in-flight window BEFORE sending (so we're ready for ACK) + // Use non-blocking tryAddInFlight since we already checked window space in ioLoop + if (inFlightWindow != null) { + LOG.debug("Adding to in-flight window [seq={}, inFlight={}, max={}]", batchSequence, inFlightWindow.getInFlightCount(), inFlightWindow.getMaxWindowSize()); + if (!inFlightWindow.tryAddInFlight(batchSequence)) { + // Should not happen since we checked hasWindowSpace before polling + throw new LineSenderException("In-flight window unexpectedly full"); + } + LOG.debug("Added to in-flight window [seq={}]", batchSequence); + } + + // Send over WebSocket + LOG.debug("Calling sendBinary [seq={}]", batchSequence); + client.sendBinary(batch.getBufferPtr(), bytes); + LOG.debug("sendBinary returned [seq={}]", batchSequence); + + // Update statistics + totalBatchesSent.incrementAndGet(); + totalBytesSent.addAndGet(bytes); + + // Transition state: SENDING -> RECYCLED + batch.markRecycled(); + + LOG.debug("Batch sent and recycled [seq={}, bufferId={}]", batchSequence, batch.getBatchId()); + } + + /** + * Checks if an error occurred in the I/O thread and throws if so. + */ + private void checkError() { + Throwable error = lastError; + if (error != null) { + throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error); + } + } + + private void failTransport(LineSenderException error) { + Throwable rootError = lastError; + if (rootError == null) { + lastError = error; + rootError = error; + } + running = false; + shuttingDown = true; + if (inFlightWindow != null) { + inFlightWindow.failAll(rootError); + } + synchronized (processingLock) { + MicrobatchBuffer dropped = pollPending(); + if (dropped != null) { + if (dropped.isSealed()) { + dropped.markSending(); + } + if (dropped.isSending()) { + dropped.markRecycled(); + } + } + processingLock.notifyAll(); + } + } + + /** + * Returns total successful acknowledgments received. + */ + public long getTotalAcks() { + return totalAcks.get(); + } + + /** + * Returns total error responses received. + */ + public long getTotalErrors() { + return totalErrors.get(); + } + + // ==================== Response Handler ==================== + + /** + * Handler for received WebSocket frames (ACKs from server). + */ + private class ResponseHandler implements WebSocketFrameHandler { + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + LineSenderException error = new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + LOG.error("Invalid ACK response payload [length={}]", payloadLen); + failTransport(error); + return; + } + + // Parse response from binary payload + if (!response.readFrom(payloadPtr, payloadLen)) { + LineSenderException error = new LineSenderException("Failed to parse ACK response"); + LOG.error("Failed to parse response"); + failTransport(error); + return; + } + + long sequence = response.getSequence(); + + if (response.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + if (inFlightWindow != null) { + int acked = inFlightWindow.acknowledgeUpTo(sequence); + if (acked > 0) { + totalAcks.addAndGet(acked); + LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); + } else { + LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); + } + } + } else { + // Error - fail the batch + String errorMessage = response.getErrorMessage(); + LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage); + + if (inFlightWindow != null) { + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + response.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + } + totalErrors.incrementAndGet(); + } + } + + @Override + public void onClose(int code, String reason) { + LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason); + failTransport(new LineSenderException("WebSocket closed by server [code=" + code + ", reason=" + reason + ']')); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java new file mode 100644 index 0000000..158ec3d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java @@ -0,0 +1,335 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Bit-level reader for ILP v4 protocol. + *

+ * This class reads bits from a buffer in LSB-first order within each byte. + * Bits are read sequentially, spanning byte boundaries as needed. + *

+ * The implementation buffers bytes to minimize memory reads. + *

+ * Usage pattern: + * 

+ * IlpV4BitReader reader = new IlpV4BitReader();
+ * reader.reset(address, length);
+ * int bit = reader.readBit();
+ * long value = reader.readBits(numBits);
+ * long signedValue = reader.readSigned(numBits);
+ *
+ */ +public class IlpV4BitReader { + + private long startAddress; + private long currentAddress; + private long endAddress; + + // Buffer for reading bits + private long bitBuffer; + // Number of bits currently available in the buffer (0-64) + private int bitsInBuffer; + // Total bits available for reading (from reset) + private long totalBitsAvailable; + // Total bits already consumed + private long totalBitsRead; + + /** + * Creates a new bit reader. Call {@link #reset} before use. + */ + public IlpV4BitReader() { + } + + /** + * Resets the reader to read from the specified memory region. + * + * @param address the starting address + * @param length the number of bytes available to read + */ + public void reset(long address, long length) { + this.startAddress = address; + this.currentAddress = address; + this.endAddress = address + length; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + this.totalBitsAvailable = length * 8L; + this.totalBitsRead = 0; + } + + /** + * Resets the reader to read from the specified byte array. + * + * @param buf the byte array + * @param offset the starting offset + * @param length the number of bytes available + */ + public void reset(byte[] buf, int offset, int length) { + // For byte array, we'll store position info differently + // This is mainly for testing - in production we use direct memory + throw new UnsupportedOperationException("Use direct memory version"); + } + + /** + * Returns the number of bits remaining to be read. + * + * @return available bits + */ + public long getAvailableBits() { + return totalBitsAvailable - totalBitsRead; + } + + /** + * Returns true if there are more bits to read. + * + * @return true if bits available + */ + public boolean hasMoreBits() { + return totalBitsRead < totalBitsAvailable; + } + + /** + * Returns the current position in bits from the start. + * + * @return bits read since reset + */ + public long getBitPosition() { + return totalBitsRead; + } + + /** + * Ensures the buffer has at least the requested number of bits. + * Loads more bytes from memory if needed. + * + * @param bitsNeeded minimum bits required in buffer + * @return true if sufficient bits available, false otherwise + */ + private boolean ensureBits(int bitsNeeded) { + while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) { + byte b = Unsafe.getUnsafe().getByte(currentAddress++); + bitBuffer |= (long) (b & 0xFF) << bitsInBuffer; + bitsInBuffer += 8; + } + return bitsInBuffer >= bitsNeeded; + } + + /** + * Reads a single bit. + * + * @return 0 or 1 + * @throws IllegalStateException if no more bits available + */ + public int readBit() { + if (totalBitsRead >= totalBitsAvailable) { + throw new IllegalStateException("bit read overflow"); + } + if (!ensureBits(1)) { + throw new IllegalStateException("bit read overflow"); + } + + int bit = (int) (bitBuffer & 1); + bitBuffer >>>= 1; + bitsInBuffer--; + totalBitsRead++; + return bit; + } + + /** + * Reads multiple bits and returns them as a long (unsigned). + *

+ * Bits are returned LSB-aligned. For example, reading 4 bits might return + * 0b1101 where bit 0 is the first bit read. + * + * @param numBits number of bits to read (1-64) + * @return the value formed by the bits (unsigned) + * @throws IllegalStateException if not enough bits available + */ + public long readBits(int numBits) { + if (numBits <= 0) { + return 0; + } + if (numBits > 64) { + throw new IllegalArgumentException("Cannot read more than 64 bits at once"); + } + if (totalBitsRead + numBits > totalBitsAvailable) { + throw new IllegalStateException("bit read overflow"); + } + + long result = 0; + int bitsRemaining = numBits; + int resultShift = 0; + + while (bitsRemaining > 0) { + if (bitsInBuffer == 0) { + if (!ensureBits(Math.min(bitsRemaining, 64))) { + throw new IllegalStateException("bit read overflow"); + } + } + + int bitsToTake = Math.min(bitsRemaining, bitsInBuffer); + long mask = bitsToTake == 64 ? -1L : (1L << bitsToTake) - 1; + result |= (bitBuffer & mask) << resultShift; + + bitBuffer >>>= bitsToTake; + bitsInBuffer -= bitsToTake; + bitsRemaining -= bitsToTake; + resultShift += bitsToTake; + } + + totalBitsRead += numBits; + return result; + } + + /** + * Reads multiple bits and interprets them as a signed value using two's complement. + * + * @param numBits number of bits to read (1-64) + * @return the signed value + * @throws IllegalStateException if not enough bits available + */ + public long readSigned(int numBits) { + long unsigned = readBits(numBits); + // Sign extend: if the high bit (bit numBits-1) is set, extend the sign + if (numBits < 64 && (unsigned & (1L << (numBits - 1))) != 0) { + // Set all bits above numBits to 1 + unsigned |= -1L << numBits; + } + return unsigned; + } + + /** + * Peeks at the next bit without consuming it. + * + * @return 0 or 1, or -1 if no more bits + */ + public int peekBit() { + if (totalBitsRead >= totalBitsAvailable) { + return -1; + } + if (!ensureBits(1)) { + return -1; + } + return (int) (bitBuffer & 1); + } + + /** + * Skips the specified number of bits. + * + * @param numBits bits to skip + * @throws IllegalStateException if not enough bits available + */ + public void skipBits(int numBits) { + if (totalBitsRead + numBits > totalBitsAvailable) { + throw new IllegalStateException("bit read overflow"); + } + + // Fast path: skip bits in current buffer + if (numBits <= bitsInBuffer) { + bitBuffer >>>= numBits; + bitsInBuffer -= numBits; + totalBitsRead += numBits; + return; + } + + // Consume all buffered bits + int bitsToSkip = numBits - bitsInBuffer; + totalBitsRead += bitsInBuffer; + bitsInBuffer = 0; + bitBuffer = 0; + + // Skip whole bytes + int bytesToSkip = bitsToSkip / 8; + currentAddress += bytesToSkip; + totalBitsRead += bytesToSkip * 8L; + + // Handle remaining bits + int remainingBits = bitsToSkip % 8; + if (remainingBits > 0) { + ensureBits(remainingBits); + bitBuffer >>>= remainingBits; + bitsInBuffer -= remainingBits; + totalBitsRead += remainingBits; + } + } + + /** + * Aligns the reader to the next byte boundary by skipping any partial bits. + * + * @throws IllegalStateException if alignment fails + */ + public void alignToByte() { + int bitsToSkip = bitsInBuffer % 8; + if (bitsToSkip != 0) { + // We need to skip the remaining bits in the current partial byte + // But since we read in byte chunks, bitsInBuffer should be a multiple of 8 + // minus what we've consumed. The remainder in the conceptual stream is: + int remainder = (int) (totalBitsRead % 8); + if (remainder != 0) { + skipBits(8 - remainder); + } + } + } + + /** + * Reads a complete byte, ensuring byte alignment first. + * + * @return the byte value (0-255) + * @throws IllegalStateException if not enough data + */ + public int readByte() { + return (int) readBits(8) & 0xFF; + } + + /** + * Reads a complete 32-bit integer in little-endian order. + * + * @return the integer value + * @throws IllegalStateException if not enough data + */ + public int readInt() { + return (int) readBits(32); + } + + /** + * Reads a complete 64-bit long in little-endian order. + * + * @return the long value + * @throws IllegalStateException if not enough data + */ + public long readLong() { + return readBits(64); + } + + /** + * Returns the current byte address being read. + * Note: This is approximate due to bit buffering. + * + * @return current address + */ + public long getCurrentAddress() { + return currentAddress; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java new file mode 100644 index 0000000..8be1600 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java @@ -0,0 +1,247 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Bit-level writer for ILP v4 protocol. + *

+ * This class writes bits to a buffer in LSB-first order within each byte. + * Bits are packed sequentially, spanning byte boundaries as needed. + *

+ * The implementation buffers up to 64 bits before flushing to the output buffer + * to minimize memory operations. All writes are to direct memory for performance. + *

+ * Usage pattern: + * 

+ * IlpV4BitWriter writer = new IlpV4BitWriter();
+ * writer.reset(address, capacity);
+ * writer.writeBits(value, numBits);
+ * writer.writeBits(value2, numBits2);
+ * writer.flush(); // must call before reading output
+ * long bytesWritten = writer.getPosition() - address;
+ *
+ */ +public class IlpV4BitWriter { + + private long startAddress; + private long currentAddress; + private long endAddress; + + // Buffer for accumulating bits before writing + private long bitBuffer; + // Number of bits currently in the buffer (0-63) + private int bitsInBuffer; + + /** + * Creates a new bit writer. Call {@link #reset} before use. + */ + public IlpV4BitWriter() { + } + + /** + * Resets the writer to write to the specified memory region. + * + * @param address the starting address + * @param capacity the maximum number of bytes to write + */ + public void reset(long address, long capacity) { + this.startAddress = address; + this.currentAddress = address; + this.endAddress = address + capacity; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + } + + /** + * Returns the current write position (address). + * Note: Call {@link #flush()} first to ensure all buffered bits are written. + * + * @return the current address after all written data + */ + public long getPosition() { + return currentAddress; + } + + /** + * Returns the number of bits that have been written (including buffered bits). + * + * @return total bits written since reset + */ + public long getTotalBitsWritten() { + return (currentAddress - startAddress) * 8L + bitsInBuffer; + } + + /** + * Writes a single bit. + * + * @param bit the bit value (0 or 1, only LSB is used) + */ + public void writeBit(int bit) { + writeBits(bit & 1, 1); + } + + /** + * Writes multiple bits from the given value. + *

+ * Bits are taken from the LSB of the value. For example, if value=0b1101 + * and numBits=4, the bits written are 1, 0, 1, 1 (LSB to MSB order). + * + * @param value the value containing the bits (LSB-aligned) + * @param numBits number of bits to write (1-64) + */ + public void writeBits(long value, int numBits) { + if (numBits <= 0 || numBits > 64) { + return; + } + + // Mask the value to only include the requested bits + if (numBits < 64) { + value &= (1L << numBits) - 1; + } + + int bitsToWrite = numBits; + + while (bitsToWrite > 0) { + // How many bits can we fit in current buffer (max 64 total) + int availableInBuffer = 64 - bitsInBuffer; + int bitsThisRound = Math.min(bitsToWrite, availableInBuffer); + + // Add bits to the buffer + long mask = bitsThisRound == 64 ? -1L : (1L << bitsThisRound) - 1; + bitBuffer |= (value & mask) << bitsInBuffer; + bitsInBuffer += bitsThisRound; + value >>>= bitsThisRound; + bitsToWrite -= bitsThisRound; + + // Flush complete bytes from the buffer + while (bitsInBuffer >= 8) { + if (currentAddress < endAddress) { + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + } + bitBuffer >>>= 8; + bitsInBuffer -= 8; + } + } + } + + /** + * Writes a signed value using two's complement representation. + * + * @param value the signed value + * @param numBits number of bits to use for the representation + */ + public void writeSigned(long value, int numBits) { + // Two's complement is automatic in Java for the bit pattern + writeBits(value, numBits); + } + + /** + * Flushes any remaining bits in the buffer to memory. + *

+ * If there are partial bits (less than 8), they are written as the last byte + * with the remaining high bits set to zero. + *

+ * Must be called before reading the output or getting the final position. + */ + public void flush() { + if (bitsInBuffer > 0 && currentAddress < endAddress) { + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + bitBuffer = 0; + bitsInBuffer = 0; + } + } + + /** + * Finishes writing and returns the number of bytes written since reset. + *

+ * This method flushes any remaining bits and returns the total byte count. + * + * @return bytes written since reset + */ + public int finish() { + flush(); + return (int) (currentAddress - startAddress); + } + + /** + * Returns the number of bits remaining in the partial byte buffer. + * This is 0 after a flush or when aligned on a byte boundary. + * + * @return bits in buffer (0-7) + */ + public int getBitsInBuffer() { + return bitsInBuffer; + } + + /** + * Aligns the writer to the next byte boundary by padding with zeros. + * If already byte-aligned, this is a no-op. + */ + public void alignToByte() { + if (bitsInBuffer > 0) { + flush(); + } + } + + /** + * Writes a complete byte, ensuring byte alignment first. + * + * @param value the byte value + */ + public void writeByte(int value) { + alignToByte(); + if (currentAddress < endAddress) { + Unsafe.getUnsafe().putByte(currentAddress++, (byte) value); + } + } + + /** + * Writes a complete 32-bit integer in little-endian order, ensuring byte alignment first. + * + * @param value the integer value + */ + public void writeInt(int value) { + alignToByte(); + if (currentAddress + 4 <= endAddress) { + Unsafe.getUnsafe().putInt(currentAddress, value); + currentAddress += 4; + } + } + + /** + * Writes a complete 64-bit long in little-endian order, ensuring byte alignment first. + * + * @param value the long value + */ + public void writeLong(long value) { + alignToByte(); + if (currentAddress + 8 <= endAddress) { + Unsafe.getUnsafe().putLong(currentAddress, value); + currentAddress += 8; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java new file mode 100644 index 0000000..cea87af --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * Represents a column definition in an ILP v4 schema. + *

+ * This class is immutable and safe for caching. + */ +public final class IlpV4ColumnDef { + private final String name; + private final byte typeCode; + private final boolean nullable; + + /** + * Creates a column definition. + * + * @param name the column name (UTF-8) + * @param typeCode the ILP v4 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable) + */ + public IlpV4ColumnDef(String name, byte typeCode) { + this.name = name; + // Extract nullable flag (high bit) and base type + this.nullable = (typeCode & 0x80) != 0; + this.typeCode = (byte) (typeCode & 0x7F); + } + + /** + * Creates a column definition with explicit nullable flag. + * + * @param name the column name + * @param typeCode the base type code (0x01-0x0F) + * @param nullable whether the column is nullable + */ + public IlpV4ColumnDef(String name, byte typeCode, boolean nullable) { + this.name = name; + this.typeCode = (byte) (typeCode & 0x7F); + this.nullable = nullable; + } + + /** + * Gets the column name. + */ + public String getName() { + return name; + } + + /** + * Gets the base type code (without nullable flag). + * + * @return type code 0x01-0x0F + */ + public byte getTypeCode() { + return typeCode; + } + + /** + * Gets the wire type code (with nullable flag if applicable). + * + * @return type code as sent on wire + */ + public byte getWireTypeCode() { + return nullable ? (byte) (typeCode | 0x80) : typeCode; + } + + /** + * Returns true if this column is nullable. + */ + public boolean isNullable() { + return nullable; + } + + /** + * Returns true if this is a fixed-width type. + */ + public boolean isFixedWidth() { + return IlpV4Constants.isFixedWidthType(typeCode); + } + + /** + * Gets the fixed width in bytes for fixed-width types. + * + * @return width in bytes, or -1 for variable-width types + */ + public int getFixedWidth() { + return IlpV4Constants.getFixedTypeSize(typeCode); + } + + /** + * Gets the type name for display purposes. + */ + public String getTypeName() { + return IlpV4Constants.getTypeName(typeCode); + } + + /** + * Validates that this column definition has a valid type code. + * + * @throws IllegalArgumentException if type code is invalid + */ + public void validate() { + // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_DECIMAL256 (0x15) + // This includes all basic types, arrays, and decimals + boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_DECIMAL256); + if (!valid) { + throw new IllegalArgumentException( + "invalid column type code: 0x" + Integer.toHexString(typeCode) + ); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IlpV4ColumnDef that = (IlpV4ColumnDef) o; + return typeCode == that.typeCode && + nullable == that.nullable && + name.equals(that.name); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + typeCode; + result = 31 * result + (nullable ? 1 : 0); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name).append(':').append(getTypeName()); + if (nullable) { + sb.append('?'); + } + return sb.toString(); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java new file mode 100644 index 0000000..7b229f6 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java @@ -0,0 +1,506 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +/** + * Constants for the ILP v4 binary protocol. + */ +public final class IlpV4Constants { + + // ==================== Magic Bytes ==================== + + /** + * Magic bytes for ILP v4 message: "ILP4" (ASCII). + */ + public static final int MAGIC_MESSAGE = 0x34504C49; // "ILP4" in little-endian + + /** + * Magic bytes for capability request: "ILP?" (ASCII). + */ + public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian + + /** + * Magic bytes for capability response: "ILP!" (ASCII). + */ + public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian + + /** + * Magic bytes for fallback response (old server): "ILP0" (ASCII). + */ + public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian + + // ==================== Header Structure ==================== + + /** + * Size of the message header in bytes. + */ + public static final int HEADER_SIZE = 12; + + /** + * Offset of magic bytes in header (4 bytes). + */ + public static final int HEADER_OFFSET_MAGIC = 0; + + /** + * Offset of version byte in header. + */ + public static final int HEADER_OFFSET_VERSION = 4; + + /** + * Offset of flags byte in header. + */ + public static final int HEADER_OFFSET_FLAGS = 5; + + /** + * Offset of table count (uint16, little-endian) in header. + */ + public static final int HEADER_OFFSET_TABLE_COUNT = 6; + + /** + * Offset of payload length (uint32, little-endian) in header. + */ + public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; + + // ==================== Protocol Version ==================== + + /** + * Current protocol version. + */ + public static final byte VERSION_1 = 1; + + // ==================== Flag Bits ==================== + + /** + * Flag bit: LZ4 compression enabled. + */ + public static final byte FLAG_LZ4 = 0x01; + + /** + * Flag bit: Zstd compression enabled. + */ + public static final byte FLAG_ZSTD = 0x02; + + /** + * Flag bit: Gorilla timestamp encoding enabled. + */ + public static final byte FLAG_GORILLA = 0x04; + + /** + * Flag bit: Delta symbol dictionary encoding enabled. + * When set, symbol columns use global IDs and send only new dictionary entries. + */ + public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08; + + /** + * Mask for compression flags (bits 0-1). + */ + public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD; + + // ==================== Column Type Codes ==================== + + /** + * Column type: BOOLEAN (1 bit per value, packed). + */ + public static final byte TYPE_BOOLEAN = 0x01; + + /** + * Column type: BYTE (int8). + */ + public static final byte TYPE_BYTE = 0x02; + + /** + * Column type: SHORT (int16, little-endian). + */ + public static final byte TYPE_SHORT = 0x03; + + /** + * Column type: INT (int32, little-endian). + */ + public static final byte TYPE_INT = 0x04; + + /** + * Column type: LONG (int64, little-endian). + */ + public static final byte TYPE_LONG = 0x05; + + /** + * Column type: FLOAT (IEEE 754 float32). + */ + public static final byte TYPE_FLOAT = 0x06; + + /** + * Column type: DOUBLE (IEEE 754 float64). + */ + public static final byte TYPE_DOUBLE = 0x07; + + /** + * Column type: STRING (length-prefixed UTF-8). + */ + public static final byte TYPE_STRING = 0x08; + + /** + * Column type: SYMBOL (dictionary-encoded string). + */ + public static final byte TYPE_SYMBOL = 0x09; + + /** + * Column type: TIMESTAMP (int64 microseconds since epoch). + * Use this for timestamps beyond nanosecond range (year > 2262). + */ + public static final byte TYPE_TIMESTAMP = 0x0A; + + /** + * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch). + * Use this for full nanosecond precision (limited to years 1677-2262). + */ + public static final byte TYPE_TIMESTAMP_NANOS = 0x10; + + /** + * Column type: DATE (int64 milliseconds since epoch). + */ + public static final byte TYPE_DATE = 0x0B; + + /** + * Column type: UUID (16 bytes, big-endian). + */ + public static final byte TYPE_UUID = 0x0C; + + /** + * Column type: LONG256 (32 bytes, big-endian). + */ + public static final byte TYPE_LONG256 = 0x0D; + + /** + * Column type: GEOHASH (varint bits + packed geohash). + */ + public static final byte TYPE_GEOHASH = 0x0E; + + /** + * Column type: VARCHAR (length-prefixed UTF-8, aux storage). + */ + public static final byte TYPE_VARCHAR = 0x0F; + + /** + * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + */ + public static final byte TYPE_DOUBLE_ARRAY = 0x11; + + /** + * Column type: LONG_ARRAY (N-dimensional array of int64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + */ + public static final byte TYPE_LONG_ARRAY = 0x12; + + /** + * Column type: DECIMAL64 (8 bytes, 18 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)] + */ + public static final byte TYPE_DECIMAL64 = 0x13; + + /** + * Column type: DECIMAL128 (16 bytes, 38 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)] + */ + public static final byte TYPE_DECIMAL128 = 0x14; + + /** + * Column type: DECIMAL256 (32 bytes, 77 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)] + */ + public static final byte TYPE_DECIMAL256 = 0x15; + + /** + * Column type: CHAR (2-byte UTF-16 code unit). + */ + public static final byte TYPE_CHAR = 0x16; + + /** + * High bit indicating nullable column. + */ + public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80; + + /** + * Mask for type code without nullable flag. + */ + public static final byte TYPE_MASK = 0x7F; + + // ==================== Schema Mode ==================== + + /** + * Schema mode: Full schema included. + */ + public static final byte SCHEMA_MODE_FULL = 0x00; + + /** + * Schema mode: Schema reference (hash lookup). + */ + public static final byte SCHEMA_MODE_REFERENCE = 0x01; + + // ==================== Response Status Codes ==================== + + /** + * Status: Batch accepted successfully. + */ + public static final byte STATUS_OK = 0x00; + + /** + * Status: Some rows failed (partial failure). + */ + public static final byte STATUS_PARTIAL = 0x01; + + /** + * Status: Schema hash not recognized. + */ + public static final byte STATUS_SCHEMA_REQUIRED = 0x02; + + /** + * Status: Column type incompatible. + */ + public static final byte STATUS_SCHEMA_MISMATCH = 0x03; + + /** + * Status: Table doesn't exist (auto-create disabled). + */ + public static final byte STATUS_TABLE_NOT_FOUND = 0x04; + + /** + * Status: Malformed message. + */ + public static final byte STATUS_PARSE_ERROR = 0x05; + + /** + * Status: Server error. + */ + public static final byte STATUS_INTERNAL_ERROR = 0x06; + + /** + * Status: Back-pressure, retry later. + */ + public static final byte STATUS_OVERLOADED = 0x07; + + // ==================== Default Limits ==================== + + /** + * Default maximum batch size in bytes (16 MB). + */ + public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024; + + /** + * Default maximum tables per batch. + */ + public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256; + + /** + * Default maximum rows per table in a batch. + */ + public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; + + /** + * Maximum columns per table (QuestDB limit). + */ + public static final int MAX_COLUMNS_PER_TABLE = 2048; + + /** + * Maximum table name length in bytes. + */ + public static final int MAX_TABLE_NAME_LENGTH = 127; + + /** + * Maximum column name length in bytes. + */ + public static final int MAX_COLUMN_NAME_LENGTH = 127; + + /** + * Default maximum string length in bytes (1 MB). + */ + public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; + + /** + * Default initial receive buffer size (64 KB). + */ + public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024; + + /** + * Maximum in-flight batches for pipelining. + */ + public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4; + + // ==================== Capability Negotiation ==================== + + /** + * Size of capability request in bytes. + */ + public static final int CAPABILITY_REQUEST_SIZE = 8; + + /** + * Size of capability response in bytes. + */ + public static final int CAPABILITY_RESPONSE_SIZE = 8; + + private IlpV4Constants() { + // utility class + } + + /** + * Returns true if the type code represents a fixed-width type. + * + * @param typeCode the column type code (without nullable flag) + * @return true if fixed-width + */ + public static boolean isFixedWidthType(byte typeCode) { + int code = typeCode & TYPE_MASK; + return code == TYPE_BOOLEAN || + code == TYPE_BYTE || + code == TYPE_SHORT || + code == TYPE_CHAR || + code == TYPE_INT || + code == TYPE_LONG || + code == TYPE_FLOAT || + code == TYPE_DOUBLE || + code == TYPE_TIMESTAMP || + code == TYPE_TIMESTAMP_NANOS || + code == TYPE_DATE || + code == TYPE_UUID || + code == TYPE_LONG256; + } + + /** + * Returns the size in bytes for fixed-width types. + * + * @param typeCode the column type code (without nullable flag) + * @return size in bytes, or -1 for variable-width types + */ + public static int getFixedTypeSize(byte typeCode) { + int code = typeCode & TYPE_MASK; + switch (code) { + case TYPE_BOOLEAN: + return 0; // Special: bit-packed + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_FLOAT: + return 4; + case TYPE_LONG: + case TYPE_DOUBLE: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + return 8; + case TYPE_UUID: + return 16; + case TYPE_LONG256: + return 32; + default: + return -1; // Variable width + } + } + + /** + * Returns a human-readable name for the type code. + * + * @param typeCode the column type code + * @return type name + */ + public static String getTypeName(byte typeCode) { + int code = typeCode & TYPE_MASK; + boolean nullable = (typeCode & TYPE_NULLABLE_FLAG) != 0; + String name; + switch (code) { + case TYPE_BOOLEAN: + name = "BOOLEAN"; + break; + case TYPE_BYTE: + name = "BYTE"; + break; + case TYPE_SHORT: + name = "SHORT"; + break; + case TYPE_CHAR: + name = "CHAR"; + break; + case TYPE_INT: + name = "INT"; + break; + case TYPE_LONG: + name = "LONG"; + break; + case TYPE_FLOAT: + name = "FLOAT"; + break; + case TYPE_DOUBLE: + name = "DOUBLE"; + break; + case TYPE_STRING: + name = "STRING"; + break; + case TYPE_SYMBOL: + name = "SYMBOL"; + break; + case TYPE_TIMESTAMP: + name = "TIMESTAMP"; + break; + case TYPE_TIMESTAMP_NANOS: + name = "TIMESTAMP_NANOS"; + break; + case TYPE_DATE: + name = "DATE"; + break; + case TYPE_UUID: + name = "UUID"; + break; + case TYPE_LONG256: + name = "LONG256"; + break; + case TYPE_GEOHASH: + name = "GEOHASH"; + break; + case TYPE_VARCHAR: + name = "VARCHAR"; + break; + case TYPE_DOUBLE_ARRAY: + name = "DOUBLE_ARRAY"; + break; + case TYPE_LONG_ARRAY: + name = "LONG_ARRAY"; + break; + case TYPE_DECIMAL64: + name = "DECIMAL64"; + break; + case TYPE_DECIMAL128: + name = "DECIMAL128"; + break; + case TYPE_DECIMAL256: + name = "DECIMAL256"; + break; + default: + name = "UNKNOWN(" + code + ")"; + } + return nullable ? name + "?" : name; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java new file mode 100644 index 0000000..2471ad4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java @@ -0,0 +1,251 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +/** + * Gorilla delta-of-delta decoder for timestamps in ILP v4 format. + *

+ * Gorilla encoding uses delta-of-delta compression where: + * 

+ * D = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
+ *
+ * if D == 0:              write '0'              (1 bit)
+ * elif D in [-63, 64]:    write '10' + 7-bit     (9 bits)
+ * elif D in [-255, 256]:  write '110' + 9-bit    (12 bits)
+ * elif D in [-2047, 2048]: write '1110' + 12-bit (16 bits)
+ * else:                   write '1111' + 32-bit  (36 bits)
+ *
+ *

+ * The decoder reads bit-packed delta-of-delta values and reconstructs + * the original timestamp sequence. + */ +public class IlpV4GorillaDecoder { + + // Bucket boundaries (two's complement signed ranges) + private static final int BUCKET_7BIT_MIN = -63; + private static final int BUCKET_7BIT_MAX = 64; + private static final int BUCKET_9BIT_MIN = -255; + private static final int BUCKET_9BIT_MAX = 256; + private static final int BUCKET_12BIT_MIN = -2047; + private static final int BUCKET_12BIT_MAX = 2048; + + private final IlpV4BitReader bitReader; + + // State for decoding + private long prevTimestamp; + private long prevDelta; + + /** + * Creates a new Gorilla decoder. + */ + public IlpV4GorillaDecoder() { + this.bitReader = new IlpV4BitReader(); + } + + /** + * Creates a decoder using an existing bit reader. + * + * @param bitReader the bit reader to use + */ + public IlpV4GorillaDecoder(IlpV4BitReader bitReader) { + this.bitReader = bitReader; + } + + /** + * Resets the decoder with the first two timestamps. + *

+ * The first two timestamps are always stored uncompressed and are used + * to establish the initial delta for subsequent compression. + * + * @param firstTimestamp the first timestamp in the sequence + * @param secondTimestamp the second timestamp in the sequence + */ + public void reset(long firstTimestamp, long secondTimestamp) { + this.prevTimestamp = secondTimestamp; + this.prevDelta = secondTimestamp - firstTimestamp; + } + + /** + * Resets the bit reader for reading encoded delta-of-deltas. + * + * @param address the address of the encoded data + * @param length the length of the encoded data in bytes + */ + public void resetReader(long address, long length) { + bitReader.reset(address, length); + } + + /** + * Decodes the next timestamp from the bit stream. + *

+ * The encoding format is: + *

    + *
  • '0' = delta-of-delta is 0 (1 bit)
    • + *
  • '10' + 7-bit signed = delta-of-delta in [-63, 64] (9 bits)
    • + *
  • '110' + 9-bit signed = delta-of-delta in [-255, 256] (12 bits)
    • + *
  • '1110' + 12-bit signed = delta-of-delta in [-2047, 2048] (16 bits)
    • + *
  • '1111' + 32-bit signed = any other delta-of-delta (36 bits)
    • + *
+ * + * @return the decoded timestamp + */ + public long decodeNext() { + long deltaOfDelta = decodeDoD(); + long delta = prevDelta + deltaOfDelta; + long timestamp = prevTimestamp + delta; + + prevDelta = delta; + prevTimestamp = timestamp; + + return timestamp; + } + + /** + * Decodes a delta-of-delta value from the bit stream. + * + * @return the delta-of-delta value + */ + private long decodeDoD() { + int bit = bitReader.readBit(); + + if (bit == 0) { + // '0' = DoD is 0 + return 0; + } + + // bit == 1, check next bit + bit = bitReader.readBit(); + if (bit == 0) { + // '10' = 7-bit signed value + return bitReader.readSigned(7); + } + + // '11', check next bit + bit = bitReader.readBit(); + if (bit == 0) { + // '110' = 9-bit signed value + return bitReader.readSigned(9); + } + + // '111', check next bit + bit = bitReader.readBit(); + if (bit == 0) { + // '1110' = 12-bit signed value + return bitReader.readSigned(12); + } + + // '1111' = 32-bit signed value + return bitReader.readSigned(32); + } + + /** + * Returns whether there are more bits available in the reader. + * + * @return true if more bits available + */ + public boolean hasMoreBits() { + return bitReader.hasMoreBits(); + } + + /** + * Returns the number of bits remaining. + * + * @return available bits + */ + public long getAvailableBits() { + return bitReader.getAvailableBits(); + } + + /** + * Returns the current bit position (bits read since reset). + * + * @return bits read + */ + public long getBitPosition() { + return bitReader.getBitPosition(); + } + + /** + * Gets the previous timestamp (for debugging/testing). + * + * @return the last decoded timestamp + */ + public long getPrevTimestamp() { + return prevTimestamp; + } + + /** + * Gets the previous delta (for debugging/testing). + * + * @return the last computed delta + */ + public long getPrevDelta() { + return prevDelta; + } + + // ==================== Static Encoding Methods (for testing) ==================== + + /** + * Determines which bucket a delta-of-delta value falls into. + * + * @param deltaOfDelta the delta-of-delta value + * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit) + */ + public static int getBucket(long deltaOfDelta) { + if (deltaOfDelta == 0) { + return 0; // 1-bit + } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) { + return 1; // 9-bit (2 prefix + 7 value) + } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) { + return 2; // 12-bit (3 prefix + 9 value) + } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) { + return 3; // 16-bit (4 prefix + 12 value) + } else { + return 4; // 36-bit (4 prefix + 32 value) + } + } + + /** + * Returns the number of bits required to encode a delta-of-delta value. + * + * @param deltaOfDelta the delta-of-delta value + * @return bits required + */ + public static int getBitsRequired(long deltaOfDelta) { + int bucket = getBucket(deltaOfDelta); + switch (bucket) { + case 0: + return 1; + case 1: + return 9; + case 2: + return 12; + case 3: + return 16; + default: + return 36; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java new file mode 100644 index 0000000..e8f0f47 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java @@ -0,0 +1,235 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Gorilla delta-of-delta encoder for timestamps in ILP v4 format. + *

+ * This encoder is used by the WebSocket encoder to compress timestamp columns. + * It uses delta-of-delta compression where: + * 

+ * DoD = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
+ *
+ * if DoD == 0:              write '0'              (1 bit)
+ * elif DoD in [-63, 64]:    write '10' + 7-bit     (9 bits)
+ * elif DoD in [-255, 256]:  write '110' + 9-bit    (12 bits)
+ * elif DoD in [-2047, 2048]: write '1110' + 12-bit (16 bits)
+ * else:                     write '1111' + 32-bit  (36 bits)
+ *
+ *

+ * The encoder writes first two timestamps uncompressed, then encodes + * remaining timestamps using delta-of-delta compression. + */ +public class IlpV4GorillaEncoder { + + private final IlpV4BitWriter bitWriter = new IlpV4BitWriter(); + + /** + * Creates a new Gorilla encoder. + */ + public IlpV4GorillaEncoder() { + } + + /** + * Encodes a delta-of-delta value using bucket selection. + *

+ * Prefix patterns are written LSB-first to match the decoder's read order: + *

    + *
  • '0' -> write bit 0
    • + *
  • '10' -> write bit 1, then bit 0 (0b01 as 2-bit value)
    • + *
  • '110' -> write bit 1, bit 1, bit 0 (0b011 as 3-bit value)
    • + *
  • '1110' -> write bit 1, bit 1, bit 1, bit 0 (0b0111 as 4-bit value)
    • + *
  • '1111' -> write bit 1, bit 1, bit 1, bit 1 (0b1111 as 4-bit value)
    • + *
+ * + * @param deltaOfDelta the delta-of-delta value to encode + */ + public void encodeDoD(long deltaOfDelta) { + int bucket = IlpV4GorillaDecoder.getBucket(deltaOfDelta); + switch (bucket) { + case 0: // DoD == 0 + bitWriter.writeBit(0); + break; + case 1: // [-63, 64] -> '10' + 7-bit + bitWriter.writeBits(0b01, 2); + bitWriter.writeSigned(deltaOfDelta, 7); + break; + case 2: // [-255, 256] -> '110' + 9-bit + bitWriter.writeBits(0b011, 3); + bitWriter.writeSigned(deltaOfDelta, 9); + break; + case 3: // [-2047, 2048] -> '1110' + 12-bit + bitWriter.writeBits(0b0111, 4); + bitWriter.writeSigned(deltaOfDelta, 12); + break; + default: // '1111' + 32-bit + bitWriter.writeBits(0b1111, 4); + bitWriter.writeSigned(deltaOfDelta, 32); + break; + } + } + + /** + * Encodes an array of timestamps to native memory using Gorilla compression. + *

+ * Format: + * 

+     * - First timestamp: int64 (8 bytes, little-endian)
+     * - Second timestamp: int64 (8 bytes, little-endian)
+     * - Remaining timestamps: bit-packed delta-of-delta
+     *
+ *

+ * Note: This method does NOT write the encoding flag byte. The caller is + * responsible for writing the ENCODING_GORILLA flag before calling this method. + * + * @param destAddress destination address in native memory + * @param capacity maximum number of bytes to write + * @param timestamps array of timestamp values + * @param count number of timestamps to encode + * @return number of bytes written + */ + public int encodeTimestamps(long destAddress, long capacity, long[] timestamps, int count) { + if (count == 0) { + return 0; + } + + int pos = 0; + + // Write first timestamp uncompressed + if (capacity < 8) { + return 0; // Not enough space + } + Unsafe.getUnsafe().putLong(destAddress, timestamps[0]); + pos = 8; + + if (count == 1) { + return pos; + } + + // Write second timestamp uncompressed + if (capacity < pos + 8) { + return pos; // Not enough space + } + Unsafe.getUnsafe().putLong(destAddress + pos, timestamps[1]); + pos += 8; + + if (count == 2) { + return pos; + } + + // Encode remaining with delta-of-delta + bitWriter.reset(destAddress + pos, capacity - pos); + long prevTs = timestamps[1]; + long prevDelta = timestamps[1] - timestamps[0]; + + for (int i = 2; i < count; i++) { + long delta = timestamps[i] - prevTs; + long dod = delta - prevDelta; + encodeDoD(dod); + prevDelta = delta; + prevTs = timestamps[i]; + } + + return pos + bitWriter.finish(); + } + + /** + * Checks if Gorilla encoding can be used for the given timestamps. + *

+ * Gorilla encoding uses 32-bit signed integers for delta-of-delta values, + * so it cannot encode timestamps where the delta-of-delta exceeds the + * 32-bit signed integer range. + * + * @param timestamps array of timestamp values + * @param count number of timestamps + * @return true if Gorilla encoding can be used, false otherwise + */ + public static boolean canUseGorilla(long[] timestamps, int count) { + if (count < 3) { + return true; // No DoD encoding needed for 0, 1, or 2 timestamps + } + + long prevDelta = timestamps[1] - timestamps[0]; + for (int i = 2; i < count; i++) { + long delta = timestamps[i] - timestamps[i - 1]; + long dod = delta - prevDelta; + if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) { + return false; + } + prevDelta = delta; + } + return true; + } + + /** + * Calculates the encoded size in bytes for Gorilla-encoded timestamps. + *

+ * Note: This does NOT include the encoding flag byte. Add 1 byte if + * the encoding flag is needed. + * + * @param timestamps array of timestamp values + * @param count number of timestamps + * @return encoded size in bytes (excluding encoding flag) + */ + public static int calculateEncodedSize(long[] timestamps, int count) { + if (count == 0) { + return 0; + } + + int size = 8; // first timestamp + + if (count == 1) { + return size; + } + + size += 8; // second timestamp + + if (count == 2) { + return size; + } + + // Calculate bits for delta-of-delta encoding + long prevTimestamp = timestamps[1]; + long prevDelta = timestamps[1] - timestamps[0]; + int totalBits = 0; + + for (int i = 2; i < count; i++) { + long delta = timestamps[i] - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); + + prevDelta = delta; + prevTimestamp = timestamps[i]; + } + + // Round up to bytes + size += (totalBits + 7) / 8; + + return size; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java new file mode 100644 index 0000000..738f2dd --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java @@ -0,0 +1,310 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Utility class for reading and writing null bitmaps in ILP v4 format. + *

+ * Null bitmap format: + *

    + *
  • Size: ceil(rowCount / 8) bytes
    • + *
  • bit[i] = 1 means row[i] is NULL
    • + *
  • Bit order: LSB first within each byte
    • + *
+ *

+ * Example: For 10 rows where rows 0, 2, 9 are null: + * 

+ * Byte 0: 0b00000101 (bits 0,2 set)
+ * Byte 1: 0b00000010 (bit 1 set, which is row 9)
+ *
+ */ +public final class IlpV4NullBitmap { + + private IlpV4NullBitmap() { + // utility class + } + + /** + * Calculates the size in bytes needed for a null bitmap. + * + * @param rowCount number of rows + * @return bitmap size in bytes + */ + public static int sizeInBytes(long rowCount) { + return (int) ((rowCount + 7) / 8); + } + + /** + * Checks if a specific row is null in the bitmap (from direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; // rowIndex / 8 + int bitIndex = rowIndex & 7; // rowIndex % 8 + byte b = Unsafe.getUnsafe().getByte(address + byteIndex); + return (b & (1 << bitIndex)) != 0; + } + + /** + * Checks if a specific row is null in the bitmap (from byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + byte b = bitmap[offset + byteIndex]; + return (b & (1 << bitIndex)) != 0; + } + + /** + * Sets a row as null in the bitmap (direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to set as null + */ + public static void setNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + long addr = address + byteIndex; + byte b = Unsafe.getUnsafe().getByte(addr); + b |= (1 << bitIndex); + Unsafe.getUnsafe().putByte(addr, b); + } + + /** + * Sets a row as null in the bitmap (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to set as null + */ + public static void setNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + bitmap[offset + byteIndex] |= (1 << bitIndex); + } + + /** + * Clears a row's null flag in the bitmap (direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to clear + */ + public static void clearNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + long addr = address + byteIndex; + byte b = Unsafe.getUnsafe().getByte(addr); + b &= ~(1 << bitIndex); + Unsafe.getUnsafe().putByte(addr, b); + } + + /** + * Clears a row's null flag in the bitmap (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to clear + */ + public static void clearNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + bitmap[offset + byteIndex] &= ~(1 << bitIndex); + } + + /** + * Counts the number of null values in the bitmap. + * + * @param address bitmap start address + * @param rowCount total number of rows + * @return count of null values + */ + public static int countNulls(long address, int rowCount) { + int count = 0; + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Count full bytes + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + count += Integer.bitCount(b & 0xFF); + } + + // Count remaining bits in last partial byte + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + count += Integer.bitCount((b & mask) & 0xFF); + } + + return count; + } + + /** + * Counts the number of null values in the bitmap (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowCount total number of rows + * @return count of null values + */ + public static int countNulls(byte[] bitmap, int offset, int rowCount) { + int count = 0; + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + for (int i = 0; i < fullBytes; i++) { + count += Integer.bitCount(bitmap[offset + i] & 0xFF); + } + + if (remainingBits > 0) { + byte b = bitmap[offset + fullBytes]; + int mask = (1 << remainingBits) - 1; + count += Integer.bitCount((b & mask) & 0xFF); + } + + return count; + } + + /** + * Checks if all rows are null. + * + * @param address bitmap start address + * @param rowCount total number of rows + * @return true if all rows are null + */ + public static boolean allNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Check full bytes (all bits should be 1) + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + if ((b & 0xFF) != 0xFF) { + return false; + } + } + + // Check remaining bits + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + if ((b & mask) != mask) { + return false; + } + } + + return true; + } + + /** + * Checks if no rows are null. + * + * @param address bitmap start address + * @param rowCount total number of rows + * @return true if no rows are null + */ + public static boolean noneNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Check full bytes + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + if (b != 0) { + return false; + } + } + + // Check remaining bits + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + if ((b & mask) != 0) { + return false; + } + } + + return true; + } + + /** + * Fills the bitmap setting all rows as null (direct memory). + * + * @param address bitmap start address + * @param rowCount total number of rows + */ + public static void fillAllNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Fill full bytes with all 1s + for (int i = 0; i < fullBytes; i++) { + Unsafe.getUnsafe().putByte(address + i, (byte) 0xFF); + } + + // Set remaining bits in last byte + if (remainingBits > 0) { + byte mask = (byte) ((1 << remainingBits) - 1); + Unsafe.getUnsafe().putByte(address + fullBytes, mask); + } + } + + /** + * Clears the bitmap setting all rows as non-null (direct memory). + * + * @param address bitmap start address + * @param rowCount total number of rows + */ + public static void fillNoneNull(long address, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + Unsafe.getUnsafe().putByte(address + i, (byte) 0); + } + } + + /** + * Clears the bitmap setting all rows as non-null (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowCount total number of rows + */ + public static void fillNoneNull(byte[] bitmap, int offset, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + bitmap[offset + i] = 0; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java new file mode 100644 index 0000000..566e2f2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java @@ -0,0 +1,574 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + + +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.DirectUtf8Sequence; +import io.questdb.client.std.str.Utf8Sequence; + +import java.nio.charset.StandardCharsets; + +/** + * XXHash64 implementation for schema hashing in ILP v4 protocol. + *

+ * The schema hash is computed over column definitions (name + type) to enable + * schema caching. When a client sends a schema reference (hash), the server + * can look up the cached schema instead of re-parsing the full schema each time. + *

+ * This is a pure Java implementation of XXHash64 based on the original algorithm + * by Yann Collet. It's optimized for small inputs typical of schema hashing. + * + * @see xxHash + */ +public final class IlpV4SchemaHash { + + // XXHash64 constants + private static final long PRIME64_1 = 0x9E3779B185EBCA87L; + private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL; + private static final long PRIME64_3 = 0x165667B19E3779F9L; + private static final long PRIME64_4 = 0x85EBCA77C2B2AE63L; + private static final long PRIME64_5 = 0x27D4EB2F165667C5L; + + // Default seed (0 for ILP v4) + private static final long DEFAULT_SEED = 0L; + + // Thread-local Hasher to avoid allocation on every computeSchemaHash call + private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); + + private IlpV4SchemaHash() { + // utility class + } + + /** + * Computes XXHash64 of a byte array. + * + * @param data the data to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data) { + return hash(data, 0, data.length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length) { + return hash(data, offset, length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region with custom seed. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length, long seed) { + long h64; + int end = offset + length; + int pos = offset; + + if (length >= 32) { + int limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, getLong(data, pos)); + pos += 8; + v2 = round(v2, getLong(data, pos)); + pos += 8; + v3 = round(v3, getLong(data, pos)); + pos += 8; + v4 = round(v4, getLong(data, pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = getLong(data, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (getInt(data, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (data[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes XXHash64 of direct memory. + * + * @param address start address + * @param length number of bytes + * @return the 64-bit hash value + */ + public static long hash(long address, long length) { + return hash(address, length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of direct memory with custom seed. + * + * @param address start address + * @param length number of bytes + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(long address, long length, long seed) { + long h64; + long end = address + length; + long pos = address; + + if (length >= 32) { + long limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v2 = round(v2, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v3 = round(v3, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v4 = round(v4, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = Unsafe.getUnsafe().getLong(pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (Unsafe.getUnsafe().getInt(pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (Unsafe.getUnsafe().getByte(pos) & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes the schema hash for ILP v4. + *

+ * Hash is computed over: for each column, hash(name_bytes + type_byte) + * This matches the spec in Appendix C. + * + * @param columnNames array of column names (UTF-8) + * @param columnTypes array of type codes + * @return the schema hash + */ + public static long computeSchemaHash(Utf8Sequence[] columnNames, byte[] columnTypes) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0; i < columnNames.length; i++) { + Utf8Sequence name = columnNames[i]; + for (int j = 0, n = name.size(); j < n; j++) { + hasher.update(name.byteAt(j)); + } + hasher.update(columnTypes[i]); + } + + return hasher.getValue(); + } + + /** + * Computes the schema hash for ILP v4 using String column names. + * Note: Iterates over String chars and converts to UTF-8 bytes directly to avoid getBytes() allocation. + * + * @param columnNames array of column names + * @param columnTypes array of type codes + * @return the schema hash + */ + public static long computeSchemaHash(String[] columnNames, byte[] columnTypes) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0; i < columnNames.length; i++) { + String name = columnNames[i]; + // Encode UTF-8 directly without allocating byte array + for (int j = 0, len = name.length(); j < len; j++) { + char c = name.charAt(j); + if (c < 0x80) { + // Single byte (ASCII) + hasher.update((byte) c); + } else if (c < 0x800) { + // Two bytes + hasher.update((byte) (0xC0 | (c >> 6))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { + // Surrogate pair (4 bytes) + char c2 = name.charAt(++j); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + // Three bytes + hasher.update((byte) (0xE0 | (c >> 12))); + hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } + } + hasher.update(columnTypes[i]); + } + + return hasher.getValue(); + } + + /** + * Computes the schema hash for ILP v4 using DirectUtf8Sequence column names. + * + * @param columnNames array of column names + * @param columnTypes array of type codes + * @return the schema hash + */ + public static long computeSchemaHash(DirectUtf8Sequence[] columnNames, byte[] columnTypes) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0; i < columnNames.length; i++) { + DirectUtf8Sequence name = columnNames[i]; + long addr = name.ptr(); + int len = name.size(); + for (int j = 0; j < len; j++) { + hasher.update(Unsafe.getUnsafe().getByte(addr + j)); + } + hasher.update(columnTypes[i]); + } + + return hasher.getValue(); + } + + /** + * Computes the schema hash directly from column buffers without intermediate arrays. + * This is the most efficient method when column data is already available. + * + * @param columns list of column buffers + * @return the schema hash + */ + public static long computeSchemaHashDirect(io.questdb.client.std.ObjList columns) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0, n = columns.size(); i < n; i++) { + IlpV4TableBuffer.ColumnBuffer col = columns.get(i); + String name = col.getName(); + // Encode UTF-8 directly without allocating byte array + for (int j = 0, len = name.length(); j < len; j++) { + char c = name.charAt(j); + if (c < 0x80) { + hasher.update((byte) c); + } else if (c < 0x800) { + hasher.update((byte) (0xC0 | (c >> 6))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { + char c2 = name.charAt(++j); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) (0xE0 | (c >> 12))); + hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } + } + // Wire type code: type | (nullable ? 0x80 : 0) + byte wireType = (byte) (col.getType() | (col.nullable ? 0x80 : 0)); + hasher.update(wireType); + } + + return hasher.getValue(); + } + + private static long round(long acc, long input) { + acc += input * PRIME64_2; + acc = Long.rotateLeft(acc, 31); + acc *= PRIME64_1; + return acc; + } + + private static long mergeRound(long acc, long val) { + val = round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; + } + + private static long avalanche(long h64) { + h64 ^= h64 >>> 33; + h64 *= PRIME64_2; + h64 ^= h64 >>> 29; + h64 *= PRIME64_3; + h64 ^= h64 >>> 32; + return h64; + } + + private static long getLong(byte[] data, int pos) { + return ((long) data[pos] & 0xFF) | + (((long) data[pos + 1] & 0xFF) << 8) | + (((long) data[pos + 2] & 0xFF) << 16) | + (((long) data[pos + 3] & 0xFF) << 24) | + (((long) data[pos + 4] & 0xFF) << 32) | + (((long) data[pos + 5] & 0xFF) << 40) | + (((long) data[pos + 6] & 0xFF) << 48) | + (((long) data[pos + 7] & 0xFF) << 56); + } + + private static int getInt(byte[] data, int pos) { + return (data[pos] & 0xFF) | + ((data[pos + 1] & 0xFF) << 8) | + ((data[pos + 2] & 0xFF) << 16) | + ((data[pos + 3] & 0xFF) << 24); + } + + /** + * Streaming hasher for incremental hash computation. + *

+ * This is useful when building the schema hash incrementally + * as columns are processed. + */ + public static class Hasher { + private long v1, v2, v3, v4; + private long totalLen; + private final byte[] buffer = new byte[32]; + private int bufferPos; + private long seed; + + public Hasher() { + reset(DEFAULT_SEED); + } + + /** + * Resets the hasher with the given seed. + * + * @param seed the hash seed + */ + public void reset(long seed) { + this.seed = seed; + v1 = seed + PRIME64_1 + PRIME64_2; + v2 = seed + PRIME64_2; + v3 = seed; + v4 = seed - PRIME64_1; + totalLen = 0; + bufferPos = 0; + } + + /** + * Updates the hash with a single byte. + * + * @param b the byte to add + */ + public void update(byte b) { + buffer[bufferPos++] = b; + totalLen++; + + if (bufferPos == 32) { + processBuffer(); + } + } + + /** + * Updates the hash with a byte array. + * + * @param data the bytes to add + */ + public void update(byte[] data) { + update(data, 0, data.length); + } + + /** + * Updates the hash with a byte array region. + * + * @param data the bytes to add + * @param offset starting offset + * @param length number of bytes + */ + public void update(byte[] data, int offset, int length) { + totalLen += length; + + // Fill buffer first + if (bufferPos > 0) { + int toCopy = Math.min(32 - bufferPos, length); + System.arraycopy(data, offset, buffer, bufferPos, toCopy); + bufferPos += toCopy; + offset += toCopy; + length -= toCopy; + + if (bufferPos == 32) { + processBuffer(); + } + } + + // Process 32-byte blocks directly + while (length >= 32) { + v1 = round(v1, getLong(data, offset)); + v2 = round(v2, getLong(data, offset + 8)); + v3 = round(v3, getLong(data, offset + 16)); + v4 = round(v4, getLong(data, offset + 24)); + offset += 32; + length -= 32; + } + + // Buffer remaining + if (length > 0) { + System.arraycopy(data, offset, buffer, 0, length); + bufferPos = length; + } + } + + /** + * Finalizes and returns the hash value. + * + * @return the 64-bit hash + */ + public long getValue() { + long h64; + + if (totalLen >= 32) { + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += totalLen; + + // Process buffered data + int pos = 0; + while (pos + 8 <= bufferPos) { + long k1 = getLong(buffer, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + if (pos + 4 <= bufferPos) { + h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + while (pos < bufferPos) { + h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + private void processBuffer() { + v1 = round(v1, getLong(buffer, 0)); + v2 = round(v2, getLong(buffer, 8)); + v3 = round(v3, getLong(buffer, 16)); + v4 = round(v4, getLong(buffer, 24)); + bufferPos = 0; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java new file mode 100644 index 0000000..6d1af59 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java @@ -0,0 +1,1424 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.std.CharSequenceIntHashMap; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; + +import java.util.Arrays; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * Buffers rows for a single table in columnar format. + *

+ * This buffer accumulates row data column by column, allowing efficient + * encoding to the ILP v4 wire format. + */ +public class IlpV4TableBuffer { + + private final String tableName; + private final ObjList columns; + private final CharSequenceIntHashMap columnNameToIndex; + private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access + private int columnAccessCursor; // tracks expected next column index + private int rowCount; + private long schemaHash; + private boolean schemaHashComputed; + private IlpV4ColumnDef[] cachedColumnDefs; + private boolean columnDefsCacheValid; + + public IlpV4TableBuffer(String tableName) { + this.tableName = tableName; + this.columns = new ObjList<>(); + this.columnNameToIndex = new CharSequenceIntHashMap(); + this.rowCount = 0; + this.schemaHash = 0; + this.schemaHashComputed = false; + this.columnDefsCacheValid = false; + } + + /** + * Returns the table name. + */ + public String getTableName() { + return tableName; + } + + /** + * Returns the number of rows buffered. + */ + public int getRowCount() { + return rowCount; + } + + /** + * Returns the number of columns. + */ + public int getColumnCount() { + return columns.size(); + } + + /** + * Returns the column at the given index. + */ + public ColumnBuffer getColumn(int index) { + return columns.get(index); + } + + /** + * Returns the column definitions (cached for efficiency). + */ + public IlpV4ColumnDef[] getColumnDefs() { + if (!columnDefsCacheValid || cachedColumnDefs == null || cachedColumnDefs.length != columns.size()) { + cachedColumnDefs = new IlpV4ColumnDef[columns.size()]; + for (int i = 0; i < columns.size(); i++) { + ColumnBuffer col = columns.get(i); + cachedColumnDefs[i] = new IlpV4ColumnDef(col.name, col.type, col.nullable); + } + columnDefsCacheValid = true; + } + return cachedColumnDefs; + } + + /** + * Gets or creates a column with the given name and type. + *

+ * Optimized for the common case where columns are accessed in the same + * order every row: a sequential cursor avoids hash map lookups entirely. + */ + public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) { + // Fast path: predict next column in sequence + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + if (candidate.name.equals(name)) { + columnAccessCursor++; + if (candidate.type != type) { + throw new IllegalArgumentException( + "Column type mismatch for " + name + ": existing=" + candidate.type + " new=" + type + ); + } + return candidate; + } + } + + // Slow path: hash map lookup + int idx = columnNameToIndex.get(name); + if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + ColumnBuffer existing = columns.get(idx); + if (existing.type != type) { + throw new IllegalArgumentException( + "Column type mismatch for " + name + ": existing=" + existing.type + " new=" + type + ); + } + return existing; + } + + // Create new column + ColumnBuffer col = new ColumnBuffer(name, type, nullable); + int index = columns.size(); + columns.add(col); + columnNameToIndex.put(name, index); + // Update fast access array + if (fastColumns == null || index >= fastColumns.length) { + int newLen = Math.max(8, index + 4); + ColumnBuffer[] newArr = new ColumnBuffer[newLen]; + if (fastColumns != null) { + System.arraycopy(fastColumns, 0, newArr, 0, index); + } + fastColumns = newArr; + } + fastColumns[index] = col; + schemaHashComputed = false; + columnDefsCacheValid = false; + return col; + } + + /** + * Advances to the next row. + *

+ * This should be called after all column values for the current row have been set. + */ + public void nextRow() { + // Reset sequential access cursor for the next row + columnAccessCursor = 0; + // Ensure all columns have the same row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + // If column wasn't set for this row, add a null + while (col.size < rowCount + 1) { + col.addNull(); + } + } + rowCount++; + } + + /** + * Cancels the current in-progress row. + *

+ * This removes any column values added since the last {@link #nextRow()} call. + * If no values have been added for the current row, this is a no-op. + */ + public void cancelCurrentRow() { + // Reset sequential access cursor + columnAccessCursor = 0; + // Truncate each column back to the committed row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + col.truncateTo(rowCount); + } + } + + /** + * Returns the schema hash for this table. + *

+ * The hash is computed to match what IlpV4Schema.computeSchemaHash() produces: + * - Uses wire type codes (with nullable bit) + * - Hash is over name bytes + type code for each column + */ + public long getSchemaHash() { + if (!schemaHashComputed) { + // Compute hash directly from column buffers without intermediate arrays + schemaHash = IlpV4SchemaHash.computeSchemaHashDirect(columns); + schemaHashComputed = true; + } + return schemaHash; + } + + /** + * Resets the buffer for reuse. + */ + public void reset() { + for (int i = 0, n = columns.size(); i < n; i++) { + fastColumns[i].reset(); + } + columnAccessCursor = 0; + rowCount = 0; + } + + /** + * Clears the buffer completely, including column definitions. + */ + public void clear() { + columns.clear(); + columnNameToIndex.clear(); + fastColumns = null; + columnAccessCursor = 0; + rowCount = 0; + schemaHash = 0; + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; + } + + /** + * Column buffer for a single column. + */ + public static class ColumnBuffer { + final String name; + final byte type; + final boolean nullable; + + private int size; // Total row count (including nulls) + private int valueCount; // Actual stored values (excludes nulls) + private int capacity; + + // Storage for different types + private boolean[] booleanValues; + private byte[] byteValues; + private short[] shortValues; + private int[] intValues; + private long[] longValues; + private float[] floatValues; + private double[] doubleValues; + private String[] stringValues; + private long[] uuidHigh; + private long[] uuidLow; + // Long256 stored as flat array: 4 longs per value (avoids inner array allocation) + private long[] long256Values; + + // Array storage (double/long arrays - variable length per row) + // Each row stores: [nDims (1B)][dim1..dimN (4B each)][flattened data] + // We track per-row metadata separately from the actual data + private byte[] arrayDims; // nDims per row + private int[] arrayShapes; // Flattened shape data (all dimensions concatenated) + private int arrayShapeOffset; // Current write offset in arrayShapes + private double[] doubleArrayData; // Flattened double values + private long[] longArrayData; // Flattened long values + private int arrayDataOffset; // Current write offset in data arrays + private int arrayRowCapacity; // Capacity for array row count + + // Null tracking - bit-packed for memory efficiency (1 bit per row vs 8 bits with boolean[]) + private long[] nullBitmapPacked; + private boolean hasNulls; + + // Symbol specific + private CharSequenceIntHashMap symbolDict; + private ObjList symbolList; + private int[] symbolIndices; + + // Global symbol IDs for delta encoding (parallel to symbolIndices) + private int[] globalSymbolIds; + private int maxGlobalSymbolId = -1; + + // Decimal storage + // All values in a decimal column must share the same scale + // For Decimal64: single long per value (64-bit unscaled) + // For Decimal128: two longs per value (128-bit unscaled: high, low) + // For Decimal256: four longs per value (256-bit unscaled: hh, hl, lh, ll) + private byte decimalScale = -1; // Shared scale for column (-1 = not set) + private long[] decimal64Values; // Decimal64: one long per value + private long[] decimal128High; // Decimal128: high 64 bits + private long[] decimal128Low; // Decimal128: low 64 bits + private long[] decimal256Hh; // Decimal256: bits 255-192 + private long[] decimal256Hl; // Decimal256: bits 191-128 + private long[] decimal256Lh; // Decimal256: bits 127-64 + private long[] decimal256Ll; // Decimal256: bits 63-0 + + public ColumnBuffer(String name, byte type, boolean nullable) { + this.name = name; + this.type = type; + this.nullable = nullable; + this.size = 0; + this.valueCount = 0; + this.capacity = 16; + this.hasNulls = false; + + allocateStorage(type, capacity); + if (nullable) { + // Bit-packed: 64 bits per long, so we need (capacity + 63) / 64 longs + nullBitmapPacked = new long[(capacity + 63) >>> 6]; + } + } + + public String getName() { + return name; + } + + public byte getType() { + return type; + } + + public int getSize() { + return size; + } + + /** + * Returns the number of actual stored values (excludes nulls). + */ + public int getValueCount() { + return valueCount; + } + + public boolean hasNulls() { + return hasNulls; + } + + /** + * Returns the bit-packed null bitmap. + * Each long contains 64 bits, bit 0 of long 0 = row 0, bit 1 of long 0 = row 1, etc. + */ + public long[] getNullBitmapPacked() { + return nullBitmapPacked; + } + + /** + * Returns the null bitmap as boolean array (for backward compatibility). + * This creates a new array, so prefer getNullBitmapPacked() for efficiency. + */ + public boolean[] getNullBitmap() { + if (nullBitmapPacked == null) { + return null; + } + boolean[] result = new boolean[size]; + for (int i = 0; i < size; i++) { + result[i] = isNull(i); + } + return result; + } + + /** + * Checks if the row at the given index is null. + */ + public boolean isNull(int index) { + if (nullBitmapPacked == null) { + return false; + } + int longIndex = index >>> 6; + int bitIndex = index & 63; + return (nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0; + } + + public boolean[] getBooleanValues() { + return booleanValues; + } + + public byte[] getByteValues() { + return byteValues; + } + + public short[] getShortValues() { + return shortValues; + } + + public int[] getIntValues() { + return intValues; + } + + public long[] getLongValues() { + return longValues; + } + + public float[] getFloatValues() { + return floatValues; + } + + public double[] getDoubleValues() { + return doubleValues; + } + + public String[] getStringValues() { + return stringValues; + } + + public long[] getUuidHigh() { + return uuidHigh; + } + + public long[] getUuidLow() { + return uuidLow; + } + + /** + * Returns Long256 values as flat array (4 longs per value). + * Use getLong256Value(index, component) for indexed access. + */ + public long[] getLong256Values() { + return long256Values; + } + + /** + * Returns a component of a Long256 value. + * @param index value index + * @param component component 0-3 + */ + public long getLong256Value(int index, int component) { + return long256Values[index * 4 + component]; + } + + // ==================== Decimal getters ==================== + + /** + * Returns the shared scale for this decimal column. + * Returns -1 if no values have been added yet. + */ + public byte getDecimalScale() { + return decimalScale; + } + + /** + * Returns the Decimal64 values (one long per value). + */ + public long[] getDecimal64Values() { + return decimal64Values; + } + + /** + * Returns the high 64 bits of Decimal128 values. + */ + public long[] getDecimal128High() { + return decimal128High; + } + + /** + * Returns the low 64 bits of Decimal128 values. + */ + public long[] getDecimal128Low() { + return decimal128Low; + } + + /** + * Returns bits 255-192 of Decimal256 values. + */ + public long[] getDecimal256Hh() { + return decimal256Hh; + } + + /** + * Returns bits 191-128 of Decimal256 values. + */ + public long[] getDecimal256Hl() { + return decimal256Hl; + } + + /** + * Returns bits 127-64 of Decimal256 values. + */ + public long[] getDecimal256Lh() { + return decimal256Lh; + } + + /** + * Returns bits 63-0 of Decimal256 values. + */ + public long[] getDecimal256Ll() { + return decimal256Ll; + } + + /** + * Returns the array dimensions per row (nDims for each row). + */ + public byte[] getArrayDims() { + return arrayDims; + } + + /** + * Returns the flattened array shapes (all dimension lengths concatenated). + */ + public int[] getArrayShapes() { + return arrayShapes; + } + + /** + * Returns the current write offset in arrayShapes. + */ + public int getArrayShapeOffset() { + return arrayShapeOffset; + } + + /** + * Returns the flattened double array data. + */ + public double[] getDoubleArrayData() { + return doubleArrayData; + } + + /** + * Returns the flattened long array data. + */ + public long[] getLongArrayData() { + return longArrayData; + } + + /** + * Returns the current write offset in the data arrays. + */ + public int getArrayDataOffset() { + return arrayDataOffset; + } + + /** + * Returns the symbol indices array (one index per value). + * Each index refers to a position in the symbol dictionary. + */ + public int[] getSymbolIndices() { + return symbolIndices; + } + + /** + * Returns the symbol dictionary as a String array. + * Index i in symbolIndices maps to symbolDictionary[i]. + */ + public String[] getSymbolDictionary() { + if (symbolList == null) { + return new String[0]; + } + String[] dict = new String[symbolList.size()]; + for (int i = 0; i < symbolList.size(); i++) { + dict[i] = symbolList.get(i); + } + return dict; + } + + /** + * Returns the size of the symbol dictionary. + */ + public int getSymbolDictionarySize() { + return symbolList == null ? 0 : symbolList.size(); + } + + /** + * Returns the global symbol IDs array for delta encoding. + * Returns null if no global IDs have been stored. + */ + public int[] getGlobalSymbolIds() { + return globalSymbolIds; + } + + /** + * Returns the maximum global symbol ID used in this column. + * Returns -1 if no symbols have been added with global IDs. + */ + public int getMaxGlobalSymbolId() { + return maxGlobalSymbolId; + } + + public void addBoolean(boolean value) { + ensureCapacity(); + booleanValues[valueCount++] = value; + size++; + } + + public void addByte(byte value) { + ensureCapacity(); + byteValues[valueCount++] = value; + size++; + } + + public void addShort(short value) { + ensureCapacity(); + shortValues[valueCount++] = value; + size++; + } + + public void addInt(int value) { + ensureCapacity(); + intValues[valueCount++] = value; + size++; + } + + public void addLong(long value) { + ensureCapacity(); + longValues[valueCount++] = value; + size++; + } + + public void addFloat(float value) { + ensureCapacity(); + floatValues[valueCount++] = value; + size++; + } + + public void addDouble(double value) { + ensureCapacity(); + doubleValues[valueCount++] = value; + size++; + } + + public void addString(String value) { + ensureCapacity(); + if (value == null && nullable) { + markNull(size); + // Null strings don't take space in the value buffer + size++; + } else { + stringValues[valueCount++] = value; + size++; + } + } + + public void addSymbol(String value) { + ensureCapacity(); + if (value == null) { + if (nullable) { + markNull(size); + } + // Null symbols don't take space in the value buffer + size++; + } else { + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + idx = symbolList.size(); + symbolDict.put(value, idx); + symbolList.add(value); + } + symbolIndices[valueCount++] = idx; + size++; + } + } + + /** + * Adds a symbol with both local dictionary and global ID tracking. + * Used for delta dictionary encoding where global IDs are shared across all columns. + * + * @param value the symbol string + * @param globalId the global ID from GlobalSymbolDictionary + */ + public void addSymbolWithGlobalId(String value, int globalId) { + ensureCapacity(); + if (value == null) { + if (nullable) { + markNull(size); + } + size++; + } else { + // Add to local dictionary (for backward compatibility with existing encoder) + int localIdx = symbolDict.get(value); + if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + localIdx = symbolList.size(); + symbolDict.put(value, localIdx); + symbolList.add(value); + } + symbolIndices[valueCount] = localIdx; + + // Also store global ID for delta encoding + if (globalSymbolIds == null) { + globalSymbolIds = new int[capacity]; + } + globalSymbolIds[valueCount] = globalId; + + // Track max global ID for this column + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; + } + + valueCount++; + size++; + } + } + + public void addUuid(long high, long low) { + ensureCapacity(); + uuidHigh[valueCount] = high; + uuidLow[valueCount] = low; + valueCount++; + size++; + } + + public void addLong256(long l0, long l1, long l2, long l3) { + ensureCapacity(); + int offset = valueCount * 4; + long256Values[offset] = l0; + long256Values[offset + 1] = l1; + long256Values[offset + 2] = l2; + long256Values[offset + 3] = l3; + valueCount++; + size++; + } + + // ==================== Decimal methods ==================== + + /** + * Adds a Decimal64 value. + * All values in a decimal column must share the same scale. + * + * @param value the Decimal64 value to add + * @throws LineSenderException if the scale doesn't match previous values + */ + public void addDecimal64(Decimal64 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + ensureCapacity(); + validateAndSetScale((byte) value.getScale()); + decimal64Values[valueCount++] = value.getValue(); + size++; + } + + /** + * Adds a Decimal128 value. + * All values in a decimal column must share the same scale. + * + * @param value the Decimal128 value to add + * @throws LineSenderException if the scale doesn't match previous values + */ + public void addDecimal128(Decimal128 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + ensureCapacity(); + validateAndSetScale((byte) value.getScale()); + decimal128High[valueCount] = value.getHigh(); + decimal128Low[valueCount] = value.getLow(); + valueCount++; + size++; + } + + /** + * Adds a Decimal256 value. + * All values in a decimal column must share the same scale. + * + * @param value the Decimal256 value to add + * @throws LineSenderException if the scale doesn't match previous values + */ + public void addDecimal256(Decimal256 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + ensureCapacity(); + validateAndSetScale((byte) value.getScale()); + decimal256Hh[valueCount] = value.getHh(); + decimal256Hl[valueCount] = value.getHl(); + decimal256Lh[valueCount] = value.getLh(); + decimal256Ll[valueCount] = value.getLl(); + valueCount++; + size++; + } + + /** + * Validates that the given scale matches the column's scale. + * If this is the first value, sets the column scale. + * + * @param scale the scale of the value being added + * @throws LineSenderException if the scale doesn't match + */ + private void validateAndSetScale(byte scale) { + if (decimalScale == -1) { + decimalScale = scale; + } else if (decimalScale != scale) { + throw new LineSenderException( + "decimal scale mismatch in column '" + name + "': expected " + + decimalScale + " but got " + scale + + ". All values in a decimal column must have the same scale." + ); + } + } + + // ==================== Array methods ==================== + + /** + * Adds a 1D double array. + */ + public void addDoubleArray(double[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (double v : values) { + doubleArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; + } + + /** + * Adds a 2D double array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addDoubleArray(double[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + // Validate rectangular shape + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + ensureArrayCapacity(2, dim0 * dim1); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (double[] row : values) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; + } + + /** + * Adds a 3D double array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addDoubleArray(double[][][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + // Validate rectangular shape + for (int i = 0; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + for (int j = 0; j < dim1; j++) { + if (values[i][j].length != dim2) { + throw new LineSenderException("irregular array shape"); + } + } + } + ensureArrayCapacity(3, dim0 * dim1 * dim2); + arrayDims[valueCount] = 3; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + arrayShapes[arrayShapeOffset++] = dim2; + for (double[][] plane : values) { + for (double[] row : plane) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + } + valueCount++; + size++; + } + + /** + * Adds a DoubleArray (N-dimensional wrapper). + * Uses a capturing approach to extract shape and data. + */ + public void addDoubleArray(DoubleArray array) { + if (array == null) { + addNull(); + return; + } + // Use a capturing ArrayBufferAppender to extract the data + ArrayCapture capture = new ArrayCapture(); + array.appendToBufPtr(capture); + + ensureArrayCapacity(capture.nDims, capture.doubleDataOffset); + arrayDims[valueCount] = capture.nDims; + for (int i = 0; i < capture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = capture.shape[i]; + } + for (int i = 0; i < capture.doubleDataOffset; i++) { + doubleArrayData[arrayDataOffset++] = capture.doubleData[i]; + } + valueCount++; + size++; + } + + /** + * Adds a 1D long array. + */ + public void addLongArray(long[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (long v : values) { + longArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; + } + + /** + * Adds a 2D long array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addLongArray(long[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + // Validate rectangular shape + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + ensureArrayCapacity(2, dim0 * dim1); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (long[] row : values) { + for (long v : row) { + longArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; + } + + /** + * Adds a 3D long array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addLongArray(long[][][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + // Validate rectangular shape + for (int i = 0; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + for (int j = 0; j < dim1; j++) { + if (values[i][j].length != dim2) { + throw new LineSenderException("irregular array shape"); + } + } + } + ensureArrayCapacity(3, dim0 * dim1 * dim2); + arrayDims[valueCount] = 3; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + arrayShapes[arrayShapeOffset++] = dim2; + for (long[][] plane : values) { + for (long[] row : plane) { + for (long v : row) { + longArrayData[arrayDataOffset++] = v; + } + } + } + valueCount++; + size++; + } + + /** + * Adds a LongArray (N-dimensional wrapper). + * Uses a capturing approach to extract shape and data. + */ + public void addLongArray(LongArray array) { + if (array == null) { + addNull(); + return; + } + // Use a capturing ArrayBufferAppender to extract the data + ArrayCapture capture = new ArrayCapture(); + array.appendToBufPtr(capture); + + ensureArrayCapacity(capture.nDims, capture.longDataOffset); + arrayDims[valueCount] = capture.nDims; + for (int i = 0; i < capture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = capture.shape[i]; + } + for (int i = 0; i < capture.longDataOffset; i++) { + longArrayData[arrayDataOffset++] = capture.longData[i]; + } + valueCount++; + size++; + } + + /** + * Ensures capacity for array storage. + * @param nDims number of dimensions for this array + * @param dataElements number of data elements + */ + private void ensureArrayCapacity(int nDims, int dataElements) { + ensureCapacity(); // For row-level capacity (arrayDims uses valueCount) + + // Ensure shape array capacity + int requiredShapeCapacity = arrayShapeOffset + nDims; + if (arrayShapes == null) { + arrayShapes = new int[Math.max(64, requiredShapeCapacity)]; + } else if (requiredShapeCapacity > arrayShapes.length) { + arrayShapes = Arrays.copyOf(arrayShapes, Math.max(arrayShapes.length * 2, requiredShapeCapacity)); + } + + // Ensure data array capacity + int requiredDataCapacity = arrayDataOffset + dataElements; + if (type == TYPE_DOUBLE_ARRAY) { + if (doubleArrayData == null) { + doubleArrayData = new double[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > doubleArrayData.length) { + doubleArrayData = Arrays.copyOf(doubleArrayData, Math.max(doubleArrayData.length * 2, requiredDataCapacity)); + } + } else if (type == TYPE_LONG_ARRAY) { + if (longArrayData == null) { + longArrayData = new long[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > longArrayData.length) { + longArrayData = Arrays.copyOf(longArrayData, Math.max(longArrayData.length * 2, requiredDataCapacity)); + } + } + } + + public void addNull() { + ensureCapacity(); + if (nullable) { + // For nullable columns, mark null in bitmap but don't store a value + markNull(size); + size++; + } else { + // For non-nullable columns, we must store a sentinel/default value + // because no null bitmap will be written + switch (type) { + case TYPE_BOOLEAN: + booleanValues[valueCount++] = false; + break; + case TYPE_BYTE: + byteValues[valueCount++] = 0; + break; + case TYPE_SHORT: + case TYPE_CHAR: + shortValues[valueCount++] = 0; + break; + case TYPE_INT: + intValues[valueCount++] = 0; + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + longValues[valueCount++] = Long.MIN_VALUE; + break; + case TYPE_FLOAT: + floatValues[valueCount++] = Float.NaN; + break; + case TYPE_DOUBLE: + doubleValues[valueCount++] = Double.NaN; + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringValues[valueCount++] = null; + break; + case TYPE_SYMBOL: + symbolIndices[valueCount++] = -1; + break; + case TYPE_UUID: + uuidHigh[valueCount] = Long.MIN_VALUE; + uuidLow[valueCount] = Long.MIN_VALUE; + valueCount++; + break; + case TYPE_LONG256: + int offset = valueCount * 4; + long256Values[offset] = Long.MIN_VALUE; + long256Values[offset + 1] = Long.MIN_VALUE; + long256Values[offset + 2] = Long.MIN_VALUE; + long256Values[offset + 3] = Long.MIN_VALUE; + valueCount++; + break; + case TYPE_DECIMAL64: + decimal64Values[valueCount++] = Decimals.DECIMAL64_NULL; + break; + case TYPE_DECIMAL128: + decimal128High[valueCount] = Decimals.DECIMAL128_HI_NULL; + decimal128Low[valueCount] = Decimals.DECIMAL128_LO_NULL; + valueCount++; + break; + case TYPE_DECIMAL256: + decimal256Hh[valueCount] = Decimals.DECIMAL256_HH_NULL; + decimal256Hl[valueCount] = Decimals.DECIMAL256_HL_NULL; + decimal256Lh[valueCount] = Decimals.DECIMAL256_LH_NULL; + decimal256Ll[valueCount] = Decimals.DECIMAL256_LL_NULL; + valueCount++; + break; + } + size++; + } + } + + private void markNull(int index) { + int longIndex = index >>> 6; + int bitIndex = index & 63; + nullBitmapPacked[longIndex] |= (1L << bitIndex); + hasNulls = true; + } + + public void reset() { + size = 0; + valueCount = 0; + hasNulls = false; + if (nullBitmapPacked != null) { + Arrays.fill(nullBitmapPacked, 0L); + } + if (symbolDict != null) { + symbolDict.clear(); + symbolList.clear(); + } + // Reset global symbol tracking + maxGlobalSymbolId = -1; + // Reset array tracking + arrayShapeOffset = 0; + arrayDataOffset = 0; + // Reset decimal scale (will be set by first non-null value) + decimalScale = -1; + } + + /** + * Truncates the column to the specified size. + * This is used to cancel uncommitted row values. + * + * @param newSize the target size (number of rows) + */ + public void truncateTo(int newSize) { + if (newSize >= size) { + return; // Nothing to truncate + } + + // Count non-null values up to newSize + int newValueCount = 0; + if (nullable && nullBitmapPacked != null) { + for (int i = 0; i < newSize; i++) { + int longIndex = i >>> 6; + int bitIndex = i & 63; + if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) == 0) { + newValueCount++; + } + } + // Clear null bits for truncated rows + for (int i = newSize; i < size; i++) { + int longIndex = i >>> 6; + int bitIndex = i & 63; + nullBitmapPacked[longIndex] &= ~(1L << bitIndex); + } + // Recompute hasNulls + hasNulls = false; + for (int i = 0; i < newSize && !hasNulls; i++) { + int longIndex = i >>> 6; + int bitIndex = i & 63; + if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0) { + hasNulls = true; + } + } + } else { + newValueCount = newSize; + } + + size = newSize; + valueCount = newValueCount; + } + + private void ensureCapacity() { + if (size >= capacity) { + int newCapacity = capacity * 2; + growStorage(type, newCapacity); + if (nullable && nullBitmapPacked != null) { + int newLongCount = (newCapacity + 63) >>> 6; + nullBitmapPacked = Arrays.copyOf(nullBitmapPacked, newLongCount); + } + capacity = newCapacity; + } + } + + private void allocateStorage(byte type, int cap) { + switch (type) { + case TYPE_BOOLEAN: + booleanValues = new boolean[cap]; + break; + case TYPE_BYTE: + byteValues = new byte[cap]; + break; + case TYPE_SHORT: + case TYPE_CHAR: + shortValues = new short[cap]; + break; + case TYPE_INT: + intValues = new int[cap]; + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + longValues = new long[cap]; + break; + case TYPE_FLOAT: + floatValues = new float[cap]; + break; + case TYPE_DOUBLE: + doubleValues = new double[cap]; + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringValues = new String[cap]; + break; + case TYPE_SYMBOL: + symbolIndices = new int[cap]; + symbolDict = new CharSequenceIntHashMap(); + symbolList = new ObjList<>(); + break; + case TYPE_UUID: + uuidHigh = new long[cap]; + uuidLow = new long[cap]; + break; + case TYPE_LONG256: + // Flat array: 4 longs per value + long256Values = new long[cap * 4]; + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + // Array types: allocate per-row tracking + // Shape and data arrays are grown dynamically in ensureArrayCapacity() + arrayDims = new byte[cap]; + arrayRowCapacity = cap; + break; + case TYPE_DECIMAL64: + decimal64Values = new long[cap]; + break; + case TYPE_DECIMAL128: + decimal128High = new long[cap]; + decimal128Low = new long[cap]; + break; + case TYPE_DECIMAL256: + decimal256Hh = new long[cap]; + decimal256Hl = new long[cap]; + decimal256Lh = new long[cap]; + decimal256Ll = new long[cap]; + break; + } + } + + private void growStorage(byte type, int newCap) { + switch (type) { + case TYPE_BOOLEAN: + booleanValues = Arrays.copyOf(booleanValues, newCap); + break; + case TYPE_BYTE: + byteValues = Arrays.copyOf(byteValues, newCap); + break; + case TYPE_SHORT: + case TYPE_CHAR: + shortValues = Arrays.copyOf(shortValues, newCap); + break; + case TYPE_INT: + intValues = Arrays.copyOf(intValues, newCap); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + longValues = Arrays.copyOf(longValues, newCap); + break; + case TYPE_FLOAT: + floatValues = Arrays.copyOf(floatValues, newCap); + break; + case TYPE_DOUBLE: + doubleValues = Arrays.copyOf(doubleValues, newCap); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringValues = Arrays.copyOf(stringValues, newCap); + break; + case TYPE_SYMBOL: + symbolIndices = Arrays.copyOf(symbolIndices, newCap); + if (globalSymbolIds != null) { + globalSymbolIds = Arrays.copyOf(globalSymbolIds, newCap); + } + break; + case TYPE_UUID: + uuidHigh = Arrays.copyOf(uuidHigh, newCap); + uuidLow = Arrays.copyOf(uuidLow, newCap); + break; + case TYPE_LONG256: + // Flat array: 4 longs per value + long256Values = Arrays.copyOf(long256Values, newCap * 4); + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + // Array types: grow per-row tracking + arrayDims = Arrays.copyOf(arrayDims, newCap); + arrayRowCapacity = newCap; + // Note: shapes and data arrays are grown in ensureArrayCapacity() + break; + case TYPE_DECIMAL64: + decimal64Values = Arrays.copyOf(decimal64Values, newCap); + break; + case TYPE_DECIMAL128: + decimal128High = Arrays.copyOf(decimal128High, newCap); + decimal128Low = Arrays.copyOf(decimal128Low, newCap); + break; + case TYPE_DECIMAL256: + decimal256Hh = Arrays.copyOf(decimal256Hh, newCap); + decimal256Hl = Arrays.copyOf(decimal256Hl, newCap); + decimal256Lh = Arrays.copyOf(decimal256Lh, newCap); + decimal256Ll = Arrays.copyOf(decimal256Ll, newCap); + break; + } + } + } + + /** + * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr(). + * This implements ArrayBufferAppender to intercept the serialization and extract + * shape and data into Java arrays for storage in ColumnBuffer. + */ + private static class ArrayCapture implements ArrayBufferAppender { + byte nDims; + int[] shape = new int[32]; // Max 32 dimensions + int shapeIndex; + double[] doubleData; + int doubleDataOffset; + long[] longData; + int longDataOffset; + + @Override + public void putByte(byte b) { + if (shapeIndex == 0) { + // First byte is nDims + nDims = b; + } + } + + @Override + public void putInt(int value) { + // Shape dimensions + if (shapeIndex < nDims) { + shape[shapeIndex++] = value; + // Once we have all dimensions, compute total elements and allocate data array + if (shapeIndex == nDims) { + int totalElements = 1; + for (int i = 0; i < nDims; i++) { + totalElements *= shape[i]; + } + // Allocate both - only one will be used + doubleData = new double[totalElements]; + longData = new long[totalElements]; + } + } + } + + @Override + public void putDouble(double value) { + if (doubleData != null && doubleDataOffset < doubleData.length) { + doubleData[doubleDataOffset++] = value; + } + } + + @Override + public void putLong(long value) { + if (longData != null && longDataOffset < longData.length) { + longData[longDataOffset++] = value; + } + } + + @Override + public void putBlockOfBytes(long from, long len) { + // This is the bulk data from the array + // The AbstractArray uses this to copy raw bytes + // We need to figure out if it's doubles or longs based on context + // For now, assume doubles (8 bytes each) since DoubleArray uses this + int count = (int) (len / 8); + if (doubleData == null) { + doubleData = new double[count]; + } + for (int i = 0; i < count; i++) { + doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L); + } + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java new file mode 100644 index 0000000..3b4fffa --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java @@ -0,0 +1,474 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Decoder for TIMESTAMP columns in ILP v4 format. + *

+ * Supports two encoding modes: + *

    + *
  • Uncompressed (0x00): array of int64 values
    • + *
  • Gorilla (0x01): delta-of-delta compressed
    • + *
+ *

+ * Gorilla format: + * 

+ * [Null bitmap if nullable]
+ * First timestamp: int64 (8 bytes, little-endian)
+ * Second timestamp: int64 (8 bytes, little-endian)
+ * Remaining timestamps: bit-packed delta-of-delta
+ *
+ */ +public final class IlpV4TimestampDecoder { + + /** + * Encoding flag for uncompressed timestamps. + */ + public static final byte ENCODING_UNCOMPRESSED = 0x00; + + /** + * Encoding flag for Gorilla-encoded timestamps. + */ + public static final byte ENCODING_GORILLA = 0x01; + + public static final IlpV4TimestampDecoder INSTANCE = new IlpV4TimestampDecoder(); + + private final IlpV4GorillaDecoder gorillaDecoder = new IlpV4GorillaDecoder(); + + private IlpV4TimestampDecoder() { + } + + /** + * Decodes timestamp column data from native memory. + * + * @param sourceAddress source address in native memory + * @param sourceLength length of source data in bytes + * @param rowCount number of rows to decode + * @param nullable whether the column is nullable + * @param sink sink to receive decoded values + * @return number of bytes consumed + */ + public int decode(long sourceAddress, int sourceLength, int rowCount, boolean nullable, ColumnSink sink) { + if (rowCount == 0) { + return 0; + } + + int offset = 0; + + // Parse null bitmap if nullable + long nullBitmapAddress = 0; + if (nullable) { + int nullBitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); + if (offset + nullBitmapSize > sourceLength) { + throw new IllegalArgumentException("insufficient data for null bitmap"); + } + nullBitmapAddress = sourceAddress + offset; + offset += nullBitmapSize; + } + + // Read encoding flag + if (offset + 1 > sourceLength) { + throw new IllegalArgumentException("insufficient data for encoding flag"); + } + byte encoding = Unsafe.getUnsafe().getByte(sourceAddress + offset); + offset++; + + if (encoding == ENCODING_UNCOMPRESSED) { + offset = decodeUncompressed(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); + } else if (encoding == ENCODING_GORILLA) { + offset = decodeGorilla(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); + } else { + throw new IllegalArgumentException("unknown timestamp encoding: " + encoding); + } + + return offset; + } + + private int decodeUncompressed(long sourceAddress, int sourceLength, int offset, int rowCount, + boolean nullable, long nullBitmapAddress, ColumnSink sink) { + // Count nulls to determine actual value count + int nullCount = 0; + if (nullable) { + nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); + } + int valueCount = rowCount - nullCount; + + // Uncompressed: valueCount * 8 bytes + int valuesSize = valueCount * 8; + if (offset + valuesSize > sourceLength) { + throw new IllegalArgumentException("insufficient data for uncompressed timestamps"); + } + + long valuesAddress = sourceAddress + offset; + int valueOffset = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + long value = Unsafe.getUnsafe().getLong(valuesAddress + (long) valueOffset * 8); + sink.putLong(i, value); + valueOffset++; + } + } + + return offset + valuesSize; + } + + private int decodeGorilla(long sourceAddress, int sourceLength, int offset, int rowCount, + boolean nullable, long nullBitmapAddress, ColumnSink sink) { + // Count nulls to determine actual value count + int nullCount = 0; + if (nullable) { + nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); + } + int valueCount = rowCount - nullCount; + + if (valueCount == 0) { + // All nulls + for (int i = 0; i < rowCount; i++) { + sink.putNull(i); + } + return offset; + } + + // First timestamp: 8 bytes + if (offset + 8 > sourceLength) { + throw new IllegalArgumentException("insufficient data for first timestamp"); + } + long firstTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); + offset += 8; + + if (valueCount == 1) { + // Only one non-null value, output it at the appropriate row position + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + sink.putLong(i, firstTimestamp); + } + } + return offset; + } + + // Second timestamp: 8 bytes + if (offset + 8 > sourceLength) { + throw new IllegalArgumentException("insufficient data for second timestamp"); + } + long secondTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); + offset += 8; + + if (valueCount == 2) { + // Two non-null values + int valueIdx = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + sink.putLong(i, valueIdx == 0 ? firstTimestamp : secondTimestamp); + valueIdx++; + } + } + return offset; + } + + // Remaining timestamps: bit-packed delta-of-delta + // Reset the Gorilla decoder with the initial state + gorillaDecoder.reset(firstTimestamp, secondTimestamp); + + // Calculate remaining bytes for bit data + int remainingBytes = sourceLength - offset; + gorillaDecoder.resetReader(sourceAddress + offset, remainingBytes); + + // Decode timestamps and distribute to rows + int valueIdx = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + long timestamp; + if (valueIdx == 0) { + timestamp = firstTimestamp; + } else if (valueIdx == 1) { + timestamp = secondTimestamp; + } else { + timestamp = gorillaDecoder.decodeNext(); + } + sink.putLong(i, timestamp); + valueIdx++; + } + } + + // Calculate how many bytes were consumed + // The bit reader has consumed some bits; round up to bytes + long bitsRead = gorillaDecoder.getAvailableBits(); + long totalBits = remainingBytes * 8L; + long bitsConsumed = totalBits - bitsRead; + int bytesConsumed = (int) ((bitsConsumed + 7) / 8); + + return offset + bytesConsumed; + } + + /** + * Returns the expected size for decoding. + * + * @param rowCount number of rows + * @param nullable whether the column is nullable + * @return expected size in bytes + */ + public int expectedSize(int rowCount, boolean nullable) { + // Minimum size: just encoding flag + uncompressed timestamps + int size = 1; // encoding flag + if (nullable) { + size += IlpV4NullBitmap.sizeInBytes(rowCount); + } + size += rowCount * 8; // worst case: uncompressed + return size; + } + + // ==================== Static Encoding Methods (for testing) ==================== + + /** + * Encodes timestamps in uncompressed format to direct memory. + * Only non-null values are written. + * + * @param destAddress destination address + * @param timestamps timestamp values + * @param nulls null flags (can be null if not nullable) + * @return address after encoded data + */ + public static long encodeUncompressed(long destAddress, long[] timestamps, boolean[] nulls) { + int rowCount = timestamps.length; + boolean nullable = nulls != null; + long pos = destAddress; + + // Write null bitmap if nullable + if (nullable) { + int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); + IlpV4NullBitmap.fillNoneNull(pos, rowCount); + for (int i = 0; i < rowCount; i++) { + if (nulls[i]) { + IlpV4NullBitmap.setNull(pos, i); + } + } + pos += bitmapSize; + } + + // Write encoding flag + Unsafe.getUnsafe().putByte(pos++, ENCODING_UNCOMPRESSED); + + // Write only non-null timestamps + for (int i = 0; i < rowCount; i++) { + if (nullable && nulls[i]) continue; + Unsafe.getUnsafe().putLong(pos, timestamps[i]); + pos += 8; + } + + return pos; + } + + /** + * Encodes timestamps in Gorilla format to direct memory. + * Only non-null values are encoded. + * + * @param destAddress destination address + * @param timestamps timestamp values + * @param nulls null flags (can be null if not nullable) + * @return address after encoded data + */ + public static long encodeGorilla(long destAddress, long[] timestamps, boolean[] nulls) { + int rowCount = timestamps.length; + boolean nullable = nulls != null; + long pos = destAddress; + + // Write null bitmap if nullable + if (nullable) { + int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); + IlpV4NullBitmap.fillNoneNull(pos, rowCount); + for (int i = 0; i < rowCount; i++) { + if (nulls[i]) { + IlpV4NullBitmap.setNull(pos, i); + } + } + pos += bitmapSize; + } + + // Count non-null values + int valueCount = 0; + for (int i = 0; i < rowCount; i++) { + if (!nullable || !nulls[i]) valueCount++; + } + + // Write encoding flag + Unsafe.getUnsafe().putByte(pos++, ENCODING_GORILLA); + + if (valueCount == 0) { + return pos; + } + + // Build array of non-null values + long[] nonNullValues = new long[valueCount]; + int idx = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && nulls[i]) continue; + nonNullValues[idx++] = timestamps[i]; + } + + // Write first timestamp + Unsafe.getUnsafe().putLong(pos, nonNullValues[0]); + pos += 8; + + if (valueCount == 1) { + return pos; + } + + // Write second timestamp + Unsafe.getUnsafe().putLong(pos, nonNullValues[1]); + pos += 8; + + if (valueCount == 2) { + return pos; + } + + // Encode remaining timestamps using Gorilla + IlpV4BitWriter bitWriter = new IlpV4BitWriter(); + bitWriter.reset(pos, 1024 * 1024); // 1MB max for bit data + + long prevTimestamp = nonNullValues[1]; + long prevDelta = nonNullValues[1] - nonNullValues[0]; + + for (int i = 2; i < valueCount; i++) { + long delta = nonNullValues[i] - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + encodeDoD(bitWriter, deltaOfDelta); + + prevDelta = delta; + prevTimestamp = nonNullValues[i]; + } + + // Flush remaining bits + int bytesWritten = bitWriter.finish(); + pos += bytesWritten; + + return pos; + } + + /** + * Encodes a delta-of-delta value to the bit writer. + *

+ * Prefix patterns are written LSB-first to match the decoder's read order: + * - '0' -> write bit 0 + * - '10' -> write bit 1, then bit 0 (0b01 as 2-bit value) + * - '110' -> write bit 1, bit 1, bit 0 (0b011 as 3-bit value) + * - '1110' -> write bit 1, bit 1, bit 1, bit 0 (0b0111 as 4-bit value) + * - '1111' -> write bit 1, bit 1, bit 1, bit 1 (0b1111 as 4-bit value) + */ + private static void encodeDoD(IlpV4BitWriter writer, long deltaOfDelta) { + if (deltaOfDelta == 0) { + // '0' = DoD is 0 + writer.writeBit(0); + } else if (deltaOfDelta >= -63 && deltaOfDelta <= 64) { + // '10' prefix: first bit read=1, second bit read=0 -> write as 0b01 (LSB-first) + writer.writeBits(0b01, 2); + writer.writeSigned(deltaOfDelta, 7); + } else if (deltaOfDelta >= -255 && deltaOfDelta <= 256) { + // '110' prefix: bits read as 1,1,0 -> write as 0b011 (LSB-first) + writer.writeBits(0b011, 3); + writer.writeSigned(deltaOfDelta, 9); + } else if (deltaOfDelta >= -2047 && deltaOfDelta <= 2048) { + // '1110' prefix: bits read as 1,1,1,0 -> write as 0b0111 (LSB-first) + writer.writeBits(0b0111, 4); + writer.writeSigned(deltaOfDelta, 12); + } else { + // '1111' prefix: bits read as 1,1,1,1 -> write as 0b1111 (LSB-first) + writer.writeBits(0b1111, 4); + writer.writeSigned(deltaOfDelta, 32); + } + } + + /** + * Calculates the encoded size in bytes for Gorilla-encoded timestamps. + * + * @param timestamps timestamp values + * @param nullable whether column is nullable + * @return encoded size in bytes + */ + public static int calculateGorillaSize(long[] timestamps, boolean nullable) { + int rowCount = timestamps.length; + int size = 0; + + if (nullable) { + size += IlpV4NullBitmap.sizeInBytes(rowCount); + } + + size += 1; // encoding flag + + if (rowCount == 0) { + return size; + } + + size += 8; // first timestamp + + if (rowCount == 1) { + return size; + } + + size += 8; // second timestamp + + if (rowCount == 2) { + return size; + } + + // Calculate bits for delta-of-delta encoding + long prevTimestamp = timestamps[1]; + long prevDelta = timestamps[1] - timestamps[0]; + int totalBits = 0; + + for (int i = 2; i < rowCount; i++) { + long delta = timestamps[i] - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); + + prevDelta = delta; + prevTimestamp = timestamps[i]; + } + + // Round up to bytes + size += (totalBits + 7) / 8; + + return size; + } + + /** + * Sink interface for receiving decoded column values. + */ + public interface ColumnSink { + void putLong(int rowIndex, long value); + void putNull(int rowIndex); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java new file mode 100644 index 0000000..cd150d4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java @@ -0,0 +1,261 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Variable-length integer encoding/decoding utilities for ILP v4 protocol. + * Uses unsigned LEB128 (Little Endian Base 128) encoding. + *

+ * The encoding scheme: + * - Values are split into 7-bit groups + * - Each byte uses the high bit (0x80) as a continuation flag + * - If high bit is set, more bytes follow + * - If high bit is clear, this is the last byte + *

+ * This implementation is designed for zero-allocation on hot paths. + */ +public final class IlpV4Varint { + + /** + * Maximum number of bytes needed to encode a 64-bit varint. + * ceil(64/7) = 10 bytes + */ + public static final int MAX_VARINT_BYTES = 10; + + /** + * Continuation bit mask - set in all bytes except the last. + */ + private static final int CONTINUATION_BIT = 0x80; + + /** + * Data mask - lower 7 bits of each byte. + */ + private static final int DATA_MASK = 0x7F; + + private IlpV4Varint() { + // utility class + } + + /** + * Calculates the number of bytes needed to encode the given value. + * + * @param value the value to measure (treated as unsigned) + * @return number of bytes needed (1-10) + */ + public static int encodedLength(long value) { + if (value == 0) { + return 1; + } + // Count leading zeros to determine the number of bits needed + int bits = 64 - Long.numberOfLeadingZeros(value); + // Each byte encodes 7 bits, round up + return (bits + 6) / 7; + } + + /** + * Encodes a long value as a varint into the given byte array. + * + * @param buf the buffer to write to + * @param pos the position to start writing + * @param value the value to encode (treated as unsigned) + * @return the new position after the encoded bytes + */ + public static int encode(byte[] buf, int pos, long value) { + while ((value & ~DATA_MASK) != 0) { + buf[pos++] = (byte) ((value & DATA_MASK) | CONTINUATION_BIT); + value >>>= 7; + } + buf[pos++] = (byte) value; + return pos; + } + + /** + * Encodes a long value as a varint to direct memory. + * + * @param address the memory address to write to + * @param value the value to encode (treated as unsigned) + * @return the new address after the encoded bytes + */ + public static long encode(long address, long value) { + while ((value & ~DATA_MASK) != 0) { + Unsafe.getUnsafe().putByte(address++, (byte) ((value & DATA_MASK) | CONTINUATION_BIT)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(address++, (byte) value); + return address; + } + + /** + * Decodes a varint from the given byte array. + * + * @param buf the buffer to read from + * @param pos the position to start reading + * @return the decoded value + * @throws IllegalArgumentException if the varint is malformed (too many bytes) + */ + public static long decode(byte[] buf, int pos) { + return decode(buf, pos, buf.length); + } + + /** + * Decodes a varint from the given byte array with bounds checking. + * + * @param buf the buffer to read from + * @param pos the position to start reading + * @param limit the maximum position to read (exclusive) + * @return the decoded value + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static long decode(byte[] buf, int pos, int limit) { + long result = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (pos >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = buf[pos++]; + result |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + return result; + } + + /** + * Decodes a varint from direct memory. + * + * @param address the memory address to read from + * @param limit the maximum address to read (exclusive) + * @return the decoded value + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static long decode(long address, long limit) { + long result = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (address >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = Unsafe.getUnsafe().getByte(address++); + result |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + return result; + } + + /** + * Result holder for decoding varints when the number of bytes consumed matters. + * This class is mutable and should be reused to avoid allocations. + */ + public static class DecodeResult { + public long value; + public int bytesRead; + + public void reset() { + value = 0; + bytesRead = 0; + } + } + + /** + * Decodes a varint from a byte array and stores both value and bytes consumed. + * + * @param buf the buffer to read from + * @param pos the position to start reading + * @param limit the maximum position to read (exclusive) + * @param result the result holder (must not be null) + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static void decode(byte[] buf, int pos, int limit, DecodeResult result) { + long value = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (pos >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = buf[pos++]; + value |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + result.value = value; + result.bytesRead = bytesRead; + } + + /** + * Decodes a varint from direct memory and stores both value and bytes consumed. + * + * @param address the memory address to read from + * @param limit the maximum address to read (exclusive) + * @param result the result holder (must not be null) + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static void decode(long address, long limit, DecodeResult result) { + long value = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (address >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = Unsafe.getUnsafe().getByte(address++); + value |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + result.value = value; + result.bytesRead = bytesRead; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java new file mode 100644 index 0000000..b0542e2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java @@ -0,0 +1,98 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +/** + * ZigZag encoding/decoding for signed integers. + *

+ * ZigZag encoding maps signed integers to unsigned integers so that + * numbers with small absolute value have small encoded values. + *

+ * The encoding works as follows: + * 

+ *  0 ->  0
+ * -1 ->  1
+ *  1 ->  2
+ * -2 ->  3
+ *  2 ->  4
+ * ...
+ *
+ *

+ * Formula: + * 

+ * encode(n) = (n << 1) ^ (n >> 63)  // for 64-bit
+ * decode(n) = (n >>> 1) ^ -(n & 1)
+ *
+ *

+ * This is useful when combined with varint encoding because small + * negative numbers like -1 become small positive numbers (1), which + * encode efficiently as varints. + */ +public final class IlpV4ZigZag { + + private IlpV4ZigZag() { + // utility class + } + + /** + * Encodes a signed 64-bit integer using ZigZag encoding. + * + * @param value the signed value to encode + * @return the ZigZag encoded value (unsigned interpretation) + */ + public static long encode(long value) { + return (value << 1) ^ (value >> 63); + } + + /** + * Decodes a ZigZag encoded 64-bit integer. + * + * @param value the ZigZag encoded value + * @return the original signed value + */ + public static long decode(long value) { + return (value >>> 1) ^ -(value & 1); + } + + /** + * Encodes a signed 32-bit integer using ZigZag encoding. + * + * @param value the signed value to encode + * @return the ZigZag encoded value (unsigned interpretation) + */ + public static int encode(int value) { + return (value << 1) ^ (value >> 31); + } + + /** + * Decodes a ZigZag encoded 32-bit integer. + * + * @param value the ZigZag encoded value + * @return the original signed value + */ + public static int decode(int value) { + return (value >>> 1) ^ -(value & 1); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java new file mode 100644 index 0000000..2cb001c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java @@ -0,0 +1,178 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +/** + * WebSocket close status codes as defined in RFC 6455. + */ +public final class WebSocketCloseCode { + /** + * Normal closure (1000). + * The connection successfully completed whatever purpose for which it was created. + */ + public static final int NORMAL_CLOSURE = 1000; + + /** + * Going away (1001). + * The endpoint is going away, e.g., server shutting down or browser navigating away. + */ + public static final int GOING_AWAY = 1001; + + /** + * Protocol error (1002). + * The endpoint is terminating the connection due to a protocol error. + */ + public static final int PROTOCOL_ERROR = 1002; + + /** + * Unsupported data (1003). + * The endpoint received a type of data it cannot accept. + */ + public static final int UNSUPPORTED_DATA = 1003; + + /** + * Reserved (1004). + * Reserved for future use. + */ + public static final int RESERVED = 1004; + + /** + * No status received (1005). + * Reserved value. MUST NOT be sent in a Close frame. + */ + public static final int NO_STATUS_RECEIVED = 1005; + + /** + * Abnormal closure (1006). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that a connection was closed abnormally. + */ + public static final int ABNORMAL_CLOSURE = 1006; + + /** + * Invalid frame payload data (1007). + * The endpoint received a message with invalid payload data. + */ + public static final int INVALID_PAYLOAD_DATA = 1007; + + /** + * Policy violation (1008). + * The endpoint received a message that violates its policy. + */ + public static final int POLICY_VIOLATION = 1008; + + /** + * Message too big (1009). + * The endpoint received a message that is too big to process. + */ + public static final int MESSAGE_TOO_BIG = 1009; + + /** + * Mandatory extension (1010). + * The client expected the server to negotiate one or more extensions. + */ + public static final int MANDATORY_EXTENSION = 1010; + + /** + * Internal server error (1011). + * The server encountered an unexpected condition that prevented it from fulfilling the request. + */ + public static final int INTERNAL_ERROR = 1011; + + /** + * TLS handshake (1015). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that the connection was closed due to TLS handshake failure. + */ + public static final int TLS_HANDSHAKE = 1015; + + private WebSocketCloseCode() { + // Constants class + } + + /** + * Checks if a close code is valid for use in a Close frame. + * Codes 1005 and 1006 are reserved and must not be sent. + * + * @param code the close code + * @return true if the code can be sent in a Close frame + */ + public static boolean isValidForSending(int code) { + if (code < 1000) { + return false; + } + if (code == NO_STATUS_RECEIVED || code == ABNORMAL_CLOSURE || code == TLS_HANDSHAKE) { + return false; + } + // 1000-2999 are defined by RFC 6455 + // 3000-3999 are reserved for libraries/frameworks + // 4000-4999 are reserved for applications + return code < 5000; + } + + /** + * Returns a human-readable description of the close code. + * + * @param code the close code + * @return the description + */ + public static String describe(int code) { + switch (code) { + case NORMAL_CLOSURE: + return "Normal Closure"; + case GOING_AWAY: + return "Going Away"; + case PROTOCOL_ERROR: + return "Protocol Error"; + case UNSUPPORTED_DATA: + return "Unsupported Data"; + case RESERVED: + return "Reserved"; + case NO_STATUS_RECEIVED: + return "No Status Received"; + case ABNORMAL_CLOSURE: + return "Abnormal Closure"; + case INVALID_PAYLOAD_DATA: + return "Invalid Payload Data"; + case POLICY_VIOLATION: + return "Policy Violation"; + case MESSAGE_TOO_BIG: + return "Message Too Big"; + case MANDATORY_EXTENSION: + return "Mandatory Extension"; + case INTERNAL_ERROR: + return "Internal Error"; + case TLS_HANDSHAKE: + return "TLS Handshake"; + default: + if (code >= 3000 && code < 4000) { + return "Library/Framework Code (" + code + ")"; + } else if (code >= 4000 && code < 5000) { + return "Application Code (" + code + ")"; + } + return "Unknown (" + code + ")"; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java new file mode 100644 index 0000000..53d02b3 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java @@ -0,0 +1,342 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +import io.questdb.client.std.Unsafe; + +/** + * Zero-allocation WebSocket frame parser. + * Parses WebSocket frames according to RFC 6455. + * + *

The parser operates on raw memory buffers and maintains minimal state. + * It can parse frames incrementally when data arrives in chunks. + * + *

Thread safety: This class is NOT thread-safe. Each connection should + * have its own parser instance. + */ +public class WebSocketFrameParser { + /** + * Initial state, waiting for frame header. + */ + public static final int STATE_HEADER = 0; + + /** + * Need more data to complete parsing. + */ + public static final int STATE_NEED_MORE = 1; + + /** + * Header parsed, need payload data. + */ + public static final int STATE_NEED_PAYLOAD = 2; + + /** + * Frame completely parsed. + */ + public static final int STATE_COMPLETE = 3; + + /** + * Error state - frame is invalid. + */ + public static final int STATE_ERROR = 4; + + // Frame header bits + private static final int FIN_BIT = 0x80; + private static final int RSV_BITS = 0x70; + private static final int OPCODE_MASK = 0x0F; + private static final int MASK_BIT = 0x80; + private static final int LENGTH_MASK = 0x7F; + + // Control frame max payload size (RFC 6455) + private static final int MAX_CONTROL_FRAME_PAYLOAD = 125; + + // Parsed frame data + private boolean fin; + private int opcode; + private boolean masked; + private int maskKey; + private long payloadLength; + private int headerSize; + + // Parser state + private int state = STATE_HEADER; + private int errorCode; + + // Configuration + private boolean serverMode = false; // If true, expect masked frames from clients + private boolean strictMode = false; // If true, reject non-minimal length encodings + + /** + * Parses a WebSocket frame from the given buffer. + * + * @param buf the start of the buffer + * @param limit the end of the buffer (exclusive) + * @return the number of bytes consumed, or 0 if more data is needed + */ + public int parse(long buf, long limit) { + long available = limit - buf; + + if (available < 2) { + state = STATE_NEED_MORE; + return 0; + } + + // Parse first two bytes + int byte0 = Unsafe.getUnsafe().getByte(buf) & 0xFF; + int byte1 = Unsafe.getUnsafe().getByte(buf + 1) & 0xFF; + + // Check reserved bits (must be 0 unless extension negotiated) + if ((byte0 & RSV_BITS) != 0) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + fin = (byte0 & FIN_BIT) != 0; + opcode = byte0 & OPCODE_MASK; + + // Validate opcode + if (!WebSocketOpcode.isValid(opcode)) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Control frames must not be fragmented + if (WebSocketOpcode.isControlFrame(opcode) && !fin) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + masked = (byte1 & MASK_BIT) != 0; + int lengthField = byte1 & LENGTH_MASK; + + // Validate masking based on mode + if (serverMode && !masked) { + // Client frames MUST be masked + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + if (!serverMode && masked) { + // Server frames MUST NOT be masked + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Calculate header size and payload length + int offset = 2; + + if (lengthField <= 125) { + payloadLength = lengthField; + } else if (lengthField == 126) { + // 16-bit extended length + if (available < 4) { + state = STATE_NEED_MORE; + return 0; + } + int high = Unsafe.getUnsafe().getByte(buf + 2) & 0xFF; + int low = Unsafe.getUnsafe().getByte(buf + 3) & 0xFF; + payloadLength = (high << 8) | low; + + // Strict mode: reject non-minimal encodings + if (strictMode && payloadLength < 126) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + offset = 4; + } else { + // 64-bit extended length + if (available < 10) { + state = STATE_NEED_MORE; + return 0; + } + payloadLength = Long.reverseBytes(Unsafe.getUnsafe().getLong(buf + 2)); + + // Strict mode: reject non-minimal encodings + if (strictMode && payloadLength <= 65535) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // MSB must be 0 (no negative lengths) + if (payloadLength < 0) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + offset = 10; + } + + // Control frames must not have payload > 125 bytes + if (WebSocketOpcode.isControlFrame(opcode) && payloadLength > MAX_CONTROL_FRAME_PAYLOAD) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Close frame with 1 byte payload is invalid (must be 0 or >= 2) + if (opcode == WebSocketOpcode.CLOSE && payloadLength == 1) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Parse mask key if present + if (masked) { + if (available < offset + 4) { + state = STATE_NEED_MORE; + return 0; + } + maskKey = Unsafe.getUnsafe().getInt(buf + offset); + offset += 4; + } else { + maskKey = 0; + } + + headerSize = offset; + + // Check if we have the complete payload + long totalFrameSize = headerSize + payloadLength; + if (available < totalFrameSize) { + state = STATE_NEED_PAYLOAD; + return headerSize; + } + + state = STATE_COMPLETE; + return (int) totalFrameSize; + } + + /** + * Unmasks the payload data in place. + * + * @param buf the start of the payload data + * @param len the length of the payload + */ + public void unmaskPayload(long buf, long len) { + if (!masked || maskKey == 0) { + return; + } + + // Process 8 bytes at a time when possible for better performance + long i = 0; + long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); + + // Process 8-byte chunks + while (i + 8 <= len) { + long value = Unsafe.getUnsafe().getLong(buf + i); + Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); + i += 8; + } + + // Process 4-byte chunk if remaining + if (i + 4 <= len) { + int value = Unsafe.getUnsafe().getInt(buf + i); + Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); + i += 4; + } + + // Process remaining bytes + while (i < len) { + byte b = Unsafe.getUnsafe().getByte(buf + i); + int shift = ((int) (i % 4)) << 3; // 0, 8, 16, or 24 + byte maskByte = (byte) ((maskKey >> shift) & 0xFF); + Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); + i++; + } + } + + /** + * Resets the parser state for parsing a new frame. + */ + public void reset() { + state = STATE_HEADER; + fin = false; + opcode = 0; + masked = false; + maskKey = 0; + payloadLength = 0; + headerSize = 0; + errorCode = 0; + } + + // Getters + + public boolean isFin() { + return fin; + } + + public int getOpcode() { + return opcode; + } + + public boolean isMasked() { + return masked; + } + + public int getMaskKey() { + return maskKey; + } + + public long getPayloadLength() { + return payloadLength; + } + + public int getHeaderSize() { + return headerSize; + } + + public int getState() { + return state; + } + + public int getErrorCode() { + return errorCode; + } + + // Setters for configuration + + public void setServerMode(boolean serverMode) { + this.serverMode = serverMode; + } + + public void setStrictMode(boolean strictMode) { + this.strictMode = strictMode; + } + + /** + * Sets the mask key for unmasking. Used in testing. + */ + public void setMaskKey(int maskKey) { + this.maskKey = maskKey; + this.masked = true; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java new file mode 100644 index 0000000..d94bcd5 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java @@ -0,0 +1,281 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; + +/** + * Zero-allocation WebSocket frame writer. + * Writes WebSocket frames according to RFC 6455. + * + *

All methods are static utilities that write directly to memory buffers. + * + *

Thread safety: This class is thread-safe as it contains no mutable state. + */ +public final class WebSocketFrameWriter { + // Frame header bits + private static final int FIN_BIT = 0x80; + private static final int MASK_BIT = 0x80; + + private WebSocketFrameWriter() { + // Static utility class + } + + /** + * Writes a WebSocket frame header to the buffer. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param masked true if the payload should be masked + * @return the number of bytes written (header size) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) { + int offset = 0; + + // First byte: FIN + opcode + int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0); + + // Second byte: MASK + payload length + int maskBit = masked ? MASK_BIT : 0; + + if (payloadLength <= 125) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength)); + } else if (payloadLength <= 65535) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF)); + } else { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127)); + Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength)); + offset += 8; + } + + return offset; + } + + /** + * Writes a WebSocket frame header with optional mask key. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param maskKey the mask key (only used if masked is true) + * @return the number of bytes written (header size including mask key) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { + int offset = writeHeader(buf, fin, opcode, payloadLength, true); + Unsafe.getUnsafe().putInt(buf + offset, maskKey); + return offset + 4; + } + + /** + * Calculates the header size for a given payload length and masking. + * + * @param payloadLength the payload length + * @param masked true if the payload will be masked + * @return the header size in bytes + */ + public static int headerSize(long payloadLength, boolean masked) { + int size; + if (payloadLength <= 125) { + size = 2; + } else if (payloadLength <= 65535) { + size = 4; + } else { + size = 10; + } + return masked ? size + 4 : size; + } + + /** + * Writes the payload for a Close frame. + * + * @param buf the buffer to write to (after the header) + * @param code the close status code + * @param reason the close reason (may be null) + * @return the number of bytes written + */ + public static int writeClosePayload(long buf, int code, String reason) { + // Write status code in network byte order (big-endian) + Unsafe.getUnsafe().putShort(buf, Short.reverseBytes((short) code)); + int offset = 2; + + // Write reason if provided + if (reason != null && !reason.isEmpty()) { + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + for (byte reasonByte : reasonBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, reasonByte); + } + } + + return offset; + } + + /** + * Writes a complete Close frame to the buffer. + * + * @param buf the buffer to write to + * @param code the close status code + * @param reason the close reason (may be null) + * @return the total number of bytes written (header + payload) + */ + public static int writeCloseFrame(long buf, int code, String reason) { + int payloadLen = 2; // status code + if (reason != null && !reason.isEmpty()) { + payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; + } + + int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); + int payloadOffset = writeClosePayload(buf + headerLen, code, reason); + + return headerLen + payloadOffset; + } + + /** + * Writes a complete Ping frame to the buffer. + * + * @param buf the buffer to write to + * @param payload the ping payload + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write + * @return the total number of bytes written + */ + public static int writePingFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.PING, payloadLen, false); + + // Copy payload + for (int i = 0; i < payloadLen; i++) { + Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); + } + + return headerLen + payloadLen; + } + + /** + * Writes a complete Pong frame to the buffer. + * + * @param buf the buffer to write to + * @param payload the pong payload (should match the received ping) + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write + * @return the total number of bytes written + */ + public static int writePongFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); + + // Copy payload + for (int i = 0; i < payloadLen; i++) { + Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); + } + + return headerLen + payloadLen; + } + + /** + * Writes a Pong frame with payload from a memory address. + * + * @param buf the buffer to write to + * @param payloadPtr pointer to the ping payload to echo + * @param payloadLen length of payload + * @return the total number of bytes written + */ + public static int writePongFrame(long buf, long payloadPtr, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); + + // Copy payload from memory + Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); + + return headerLen + payloadLen; + } + + /** + * Writes a binary frame with payload from a memory address. + * + * @param buf the buffer to write to + * @param payloadPtr pointer to the payload data + * @param payloadLen length of payload + * @return the total number of bytes written + */ + public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + + // Copy payload from memory + Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); + + return headerLen + payloadLen; + } + + /** + * Writes a binary frame header only (for when payload is written separately). + * + * @param buf the buffer to write to + * @param payloadLen length of payload that will follow + * @return the header size in bytes + */ + public static int writeBinaryFrameHeader(long buf, int payloadLen) { + return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + } + + /** + * Masks payload data in place using XOR with the given mask key. + * + * @param buf the payload buffer + * @param len the payload length + * @param maskKey the 4-byte mask key + */ + public static void maskPayload(long buf, long len, int maskKey) { + // Process 8 bytes at a time when possible + long i = 0; + long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); + + // Process 8-byte chunks + while (i + 8 <= len) { + long value = Unsafe.getUnsafe().getLong(buf + i); + Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); + i += 8; + } + + // Process 4-byte chunk if remaining + if (i + 4 <= len) { + int value = Unsafe.getUnsafe().getInt(buf + i); + Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); + i += 4; + } + + // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation + while (i < len) { + byte b = Unsafe.getUnsafe().getByte(buf + i); + int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF; + Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); + i++; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java new file mode 100644 index 0000000..5248942 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java @@ -0,0 +1,421 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8Sequence; +import io.questdb.client.std.str.Utf8String; +import io.questdb.client.std.str.Utf8s; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; + +/** + * WebSocket handshake processing as defined in RFC 6455. + * Provides utilities for validating WebSocket upgrade requests and + * generating proper handshake responses. + */ +public final class WebSocketHandshake { + /** + * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. + */ + public static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + + /** + * The required WebSocket version (RFC 6455). + */ + public static final int WEBSOCKET_VERSION = 13; + + // Header names (case-insensitive) + public static final Utf8String HEADER_UPGRADE = new Utf8String("Upgrade"); + public static final Utf8String HEADER_CONNECTION = new Utf8String("Connection"); + public static final Utf8String HEADER_SEC_WEBSOCKET_KEY = new Utf8String("Sec-WebSocket-Key"); + public static final Utf8String HEADER_SEC_WEBSOCKET_VERSION = new Utf8String("Sec-WebSocket-Version"); + public static final Utf8String HEADER_SEC_WEBSOCKET_PROTOCOL = new Utf8String("Sec-WebSocket-Protocol"); + public static final Utf8String HEADER_SEC_WEBSOCKET_ACCEPT = new Utf8String("Sec-WebSocket-Accept"); + + // Header values + public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); + public static final Utf8String VALUE_UPGRADE = new Utf8String("upgrade"); + + // Response template + private static final byte[] RESPONSE_PREFIX = + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ".getBytes(StandardCharsets.US_ASCII); + private static final byte[] RESPONSE_SUFFIX = "\r\n\r\n".getBytes(StandardCharsets.US_ASCII); + + // Thread-local SHA-1 digest for computing Sec-WebSocket-Accept + private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-1 not available", e); + } + }); + + private WebSocketHandshake() { + // Static utility class + } + + /** + * Checks if the given header indicates a WebSocket upgrade request. + * + * @param upgradeHeader the value of the Upgrade header + * @return true if this is a WebSocket upgrade request + */ + public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { + return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); + } + + /** + * Checks if the Connection header contains "upgrade". + * + * @param connectionHeader the value of the Connection header + * @return true if the connection should be upgraded + */ + public static boolean isConnectionUpgrade(Utf8Sequence connectionHeader) { + if (connectionHeader == null) { + return false; + } + // Connection header may contain multiple values, e.g., "keep-alive, Upgrade" + // Perform case-insensitive substring search + return containsIgnoreCaseAscii(connectionHeader, VALUE_UPGRADE); + } + + /** + * Checks if the sequence contains the given substring (case-insensitive). + */ + private static boolean containsIgnoreCaseAscii(Utf8Sequence seq, Utf8Sequence substring) { + int seqLen = seq.size(); + int subLen = substring.size(); + + if (subLen > seqLen) { + return false; + } + if (subLen == 0) { + return true; + } + + outer: + for (int i = 0; i <= seqLen - subLen; i++) { + for (int j = 0; j < subLen; j++) { + byte a = seq.byteAt(i + j); + byte b = substring.byteAt(j); + // Convert to lowercase for comparison + if (a >= 'A' && a <= 'Z') { + a = (byte) (a + 32); + } + if (b >= 'A' && b <= 'Z') { + b = (byte) (b + 32); + } + if (a != b) { + continue outer; + } + } + return true; + } + return false; + } + + /** + * Validates the WebSocket version. + * + * @param versionHeader the Sec-WebSocket-Version header value + * @return true if the version is valid (13) + */ + public static boolean isValidVersion(Utf8Sequence versionHeader) { + if (versionHeader == null || versionHeader.size() == 0) { + return false; + } + // Parse the version number + try { + int version = 0; + for (int i = 0; i < versionHeader.size(); i++) { + byte b = versionHeader.byteAt(i); + if (b < '0' || b > '9') { + return false; + } + version = version * 10 + (b - '0'); + } + return version == WEBSOCKET_VERSION; + } catch (Exception e) { + return false; + } + } + + /** + * Validates the Sec-WebSocket-Key header. + * The key must be a base64-encoded 16-byte value. + * + * @param key the Sec-WebSocket-Key header value + * @return true if the key is valid + */ + public static boolean isValidKey(Utf8Sequence key) { + if (key == null) { + return false; + } + // Base64-encoded 16-byte value should be exactly 24 characters + // (16 bytes = 128 bits = 22 base64 chars + 2 padding = 24) + int size = key.size(); + if (size != 24) { + return false; + } + // Basic validation: check that all characters are valid base64 + for (int i = 0; i < size; i++) { + byte b = key.byteAt(i); + boolean valid = (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || + (b >= '0' && b <= '9') || b == '+' || b == '/' || b == '='; + if (!valid) { + return false; + } + } + return true; + } + + /** + * Computes the Sec-WebSocket-Accept value for the given key. + * + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response + */ + public static String computeAcceptKey(Utf8Sequence key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + byte[] keyBytes = new byte[key.size()]; + for (int i = 0; i < key.size(); i++) { + keyBytes[i] = key.byteAt(i); + } + sha1.update(keyBytes); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); + } + + /** + * Computes the Sec-WebSocket-Accept value for the given key string. + * + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response + */ + public static String computeAcceptKey(String key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + sha1.update(key.getBytes(StandardCharsets.US_ASCII)); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); + } + + /** + * Writes the WebSocket handshake response to the given buffer. + * + * @param buf the buffer to write to + * @param acceptKey the computed Sec-WebSocket-Accept value + * @return the number of bytes written + */ + public static int writeResponse(long buf, String acceptKey) { + int offset = 0; + + // Write prefix + for (byte b : RESPONSE_PREFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write accept key + byte[] acceptBytes = acceptKey.getBytes(StandardCharsets.US_ASCII); + for (byte b : acceptBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write suffix + for (byte b : RESPONSE_SUFFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Returns the size of the handshake response for the given accept key. + * + * @param acceptKey the computed accept key + * @return the total response size in bytes + */ + public static int responseSize(String acceptKey) { + return RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + } + + /** + * Writes the WebSocket handshake response with an optional subprotocol. + * + * @param buf the buffer to write to + * @param acceptKey the computed Sec-WebSocket-Accept value + * @param protocol the negotiated subprotocol (may be null or empty) + * @return the number of bytes written + */ + public static int writeResponseWithProtocol(long buf, String acceptKey, String protocol) { + int offset = 0; + + // Write prefix + for (byte b : RESPONSE_PREFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write accept key + byte[] acceptBytes = acceptKey.getBytes(StandardCharsets.US_ASCII); + for (byte b : acceptBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write protocol header if present + if (protocol != null && !protocol.isEmpty()) { + byte[] protocolHeader = ("\r\nSec-WebSocket-Protocol: " + protocol).getBytes(StandardCharsets.US_ASCII); + for (byte b : protocolHeader) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + } + + // Write suffix + for (byte b : RESPONSE_SUFFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Returns the size of the handshake response with an optional subprotocol. + * + * @param acceptKey the computed accept key + * @param protocol the negotiated subprotocol (may be null or empty) + * @return the total response size in bytes + */ + public static int responseSizeWithProtocol(String acceptKey, String protocol) { + int size = RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + if (protocol != null && !protocol.isEmpty()) { + size += "\r\nSec-WebSocket-Protocol: ".length() + protocol.length(); + } + return size; + } + + /** + * Writes a 400 Bad Request response. + * + * @param buf the buffer to write to + * @param reason the reason for the bad request + * @return the number of bytes written + */ + public static int writeBadRequestResponse(long buf, String reason) { + int offset = 0; + + byte[] statusLine = "HTTP/1.1 400 Bad Request\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : statusLine) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] contentType = "Content-Type: text/plain\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : contentType) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] reasonBytes = reason != null ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] contentLength = ("Content-Length: " + reasonBytes.length + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII); + for (byte b : contentLength) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + for (byte b : reasonBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Writes a 426 Upgrade Required response indicating unsupported WebSocket version. + * + * @param buf the buffer to write to + * @return the number of bytes written + */ + public static int writeVersionNotSupportedResponse(long buf) { + int offset = 0; + + byte[] statusLine = "HTTP/1.1 426 Upgrade Required\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : statusLine) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] versionHeader = "Sec-WebSocket-Version: 13\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : versionHeader) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] contentLength = "Content-Length: 0\r\n\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : contentLength) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Validates all required headers for a WebSocket upgrade request. + * + * @param upgradeHeader the Upgrade header value + * @param connectionHeader the Connection header value + * @param keyHeader the Sec-WebSocket-Key header value + * @param versionHeader the Sec-WebSocket-Version header value + * @return null if valid, or an error message describing the problem + */ + public static String validate( + Utf8Sequence upgradeHeader, + Utf8Sequence connectionHeader, + Utf8Sequence keyHeader, + Utf8Sequence versionHeader + ) { + if (!isWebSocketUpgrade(upgradeHeader)) { + return "Missing or invalid Upgrade header"; + } + if (!isConnectionUpgrade(connectionHeader)) { + return "Missing or invalid Connection header"; + } + if (!isValidKey(keyHeader)) { + return "Missing or invalid Sec-WebSocket-Key header"; + } + if (!isValidVersion(versionHeader)) { + return "Unsupported WebSocket version"; + } + return null; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java new file mode 100644 index 0000000..03fe188 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java @@ -0,0 +1,136 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +/** + * WebSocket frame opcodes as defined in RFC 6455. + */ +public final class WebSocketOpcode { + /** + * Continuation frame (0x0). + * Used for fragmented messages after the initial frame. + */ + public static final int CONTINUATION = 0x00; + + /** + * Text frame (0x1). + * Payload is UTF-8 encoded text. + */ + public static final int TEXT = 0x01; + + /** + * Binary frame (0x2). + * Payload is arbitrary binary data. + */ + public static final int BINARY = 0x02; + + // Reserved non-control frames: 0x3-0x7 + + /** + * Connection close frame (0x8). + * Indicates that the endpoint wants to close the connection. + */ + public static final int CLOSE = 0x08; + + /** + * Ping frame (0x9). + * Used for keep-alive and connection health checks. + */ + public static final int PING = 0x09; + + /** + * Pong frame (0xA). + * Response to a ping frame. + */ + public static final int PONG = 0x0A; + + // Reserved control frames: 0xB-0xF + + private WebSocketOpcode() { + // Constants class + } + + /** + * Checks if the opcode is a control frame. + * Control frames are CLOSE (0x8), PING (0x9), and PONG (0xA). + * + * @param opcode the opcode to check + * @return true if the opcode is a control frame + */ + public static boolean isControlFrame(int opcode) { + return (opcode & 0x08) != 0; + } + + /** + * Checks if the opcode is a data frame. + * Data frames are CONTINUATION (0x0), TEXT (0x1), and BINARY (0x2). + * + * @param opcode the opcode to check + * @return true if the opcode is a data frame + */ + public static boolean isDataFrame(int opcode) { + return opcode <= 0x02; + } + + /** + * Checks if the opcode is valid according to RFC 6455. + * + * @param opcode the opcode to check + * @return true if the opcode is valid + */ + public static boolean isValid(int opcode) { + return opcode == CONTINUATION + || opcode == TEXT + || opcode == BINARY + || opcode == CLOSE + || opcode == PING + || opcode == PONG; + } + + /** + * Returns a human-readable name for the opcode. + * + * @param opcode the opcode + * @return the opcode name + */ + public static String name(int opcode) { + switch (opcode) { + case CONTINUATION: + return "CONTINUATION"; + case TEXT: + return "TEXT"; + case BINARY: + return "BINARY"; + case CLOSE: + return "CLOSE"; + case PING: + return "PING"; + case PONG: + return "PONG"; + default: + return "UNKNOWN(" + opcode + ")"; + } + } +} diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java new file mode 100644 index 0000000..d299423 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -0,0 +1,209 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import org.jetbrains.annotations.NotNull; + +import java.util.Arrays; + + +public class CharSequenceIntHashMap extends AbstractCharSequenceHashSet { + public static final int NO_ENTRY_VALUE = -1; + private final ObjList list; + private final int noEntryValue; + private int[] values; + + public CharSequenceIntHashMap() { + this(8); + } + + public CharSequenceIntHashMap(int initialCapacity) { + this(initialCapacity, 0.4, NO_ENTRY_VALUE); + } + + public CharSequenceIntHashMap(int initialCapacity, double loadFactor, int noEntryValue) { + super(initialCapacity, loadFactor); + this.noEntryValue = noEntryValue; + this.list = new ObjList<>(capacity); + values = new int[keys.length]; + clear(); + } + + @Override + public final void clear() { + super.clear(); + list.clear(); + Arrays.fill(values, noEntryValue); + } + + public int get(@NotNull CharSequence key) { + return valueAt(keyIndex(key)); + } + + public void inc(@NotNull CharSequence key) { + int index = keyIndex(key); + if (index < 0) { + values[-index - 1]++; + } else { + putAt0(index, Chars.toString(key), 1); + } + } + + public ObjList keys() { + return list; + } + + public boolean put(@NotNull CharSequence key, int value) { + return putAt(keyIndex(key), key, value); + } + + public void putAll(@NotNull CharSequenceIntHashMap other) { + CharSequence[] otherKeys = other.keys; + int[] otherValues = other.values; + for (int i = 0, n = otherKeys.length; i < n; i++) { + if (otherKeys[i] != noEntryKey) { + put(otherKeys[i], otherValues[i]); + } + } + } + + public boolean putAt(int index, @NotNull CharSequence key, int value) { + if (index < 0) { + values[-index - 1] = value; + return false; + } + final String keyString = Chars.toString(key); + putAt0(index, keyString, value); + list.add(keyString); + return true; + } + + public void putIfAbsent(@NotNull CharSequence key, int value) { + int index = keyIndex(key); + if (index > -1) { + String keyString = Chars.toString(key); + putAt0(index, keyString, value); + list.add(keyString); + } + } + + public void removeAt(int index) { + if (index < 0) { + int from = -index - 1; + CharSequence key = keys[from]; + erase(from); + free++; + + // after we have freed up a slot + // consider non-empty keys directly below + // they may have been a direct hit but because + // directly hit slot wasn't empty these keys would + // have moved. + // + // After slot is freed these keys require re-hash + from = (from + 1) & mask; + for ( + CharSequence k = keys[from]; + k != noEntryKey; + from = (from + 1) & mask, k = keys[from] + ) { + int idealHit = Hash.spread(Chars.hashCode(k)) & mask; + if (idealHit != from) { + int to; + if (keys[idealHit] != noEntryKey) { + to = probe0(k, idealHit); + } else { + to = idealHit; + } + + if (to > -1) { + move(from, to); + } + } + } + + list.remove(key); + } + } + + public int valueAt(int index) { + int index1 = -index - 1; + return index < 0 ? values[index1] : noEntryValue; + } + + public int valueQuick(int index) { + return get(list.getQuick(index)); + } + + private void putAt0(int index, CharSequence key, int value) { + keys[index] = key; + values[index] = value; + if (--free == 0) { + rehash(); + } + } + + private void rehash() { + int[] oldValues = values; + CharSequence[] oldKeys = keys; + int size = capacity - free; + capacity = capacity * 2; + free = capacity - size; + mask = Numbers.ceilPow2((int) (capacity / loadFactor)) - 1; + this.keys = new CharSequence[mask + 1]; + this.values = new int[mask + 1]; + for (int i = oldKeys.length - 1; i > -1; i--) { + CharSequence key = oldKeys[i]; + if (key != null) { + final int index = keyIndex(key); + keys[index] = key; + values[index] = oldValues[i]; + } + } + } + + private void erase(int index) { + keys[index] = noEntryKey; + values[index] = noEntryValue; + } + + private void move(int from, int to) { + keys[to] = keys[from]; + values[to] = values[from]; + erase(from); + } + + private int probe0(CharSequence key, int index) { + do { + index = (index + 1) & mask; + if (keys[index] == noEntryKey) { + return index; + } + if (Chars.equals(key, keys[index])) { + return -index - 1; + } + } while (true); + } +} diff --git a/core/src/main/java/io/questdb/client/std/LongHashSet.java b/core/src/main/java/io/questdb/client/std/LongHashSet.java new file mode 100644 index 0000000..4ec2648 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/LongHashSet.java @@ -0,0 +1,155 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import io.questdb.client.std.str.CharSink; +import io.questdb.client.std.str.Sinkable; +import org.jetbrains.annotations.NotNull; + +import java.util.Arrays; + + +public class LongHashSet extends AbstractLongHashSet implements Sinkable { + + public static final double DEFAULT_LOAD_FACTOR = 0.4; + private static final int MIN_INITIAL_CAPACITY = 16; + private final LongList list; + + public LongHashSet() { + this(MIN_INITIAL_CAPACITY); + } + + public LongHashSet(int initialCapacity) { + this(initialCapacity, DEFAULT_LOAD_FACTOR, noEntryKey); + } + + public LongHashSet(int initialCapacity, double loadFactor, long noKeyValue) { + super(initialCapacity, loadFactor, noKeyValue); + list = new LongList(free); + clear(); + } + + /** + * Adds key to hash set preserving key uniqueness. + * + * @param key key to be added. + * @return false if key is already in the set and true otherwise. + */ + public boolean add(long key) { + int index = keyIndex(key); + if (index < 0) { + return false; + } + + addAt(index, key); + return true; + } + + public void addAt(int index, long key) { + keys[index] = key; + list.add(key); + if (--free < 1) { + rehash(); + } + } + + public final void clear() { + free = capacity; + Arrays.fill(keys, noEntryKeyValue); + list.clear(); + } + + public boolean contains(long key) { + return keyIndex(key) < 0; + } + + public long get(int index) { + return list.getQuick(index); + } + + public long getLast() { + return list.getLast(); + } + + public void removeAt(int index) { + if (index < 0) { + long key = keys[-index - 1]; + super.removeAt(index); + listRemove(key); + } + } + + @Override + public void toSink(@NotNull CharSink sink) { + list.toSink(sink); + } + + @Override + public String toString() { + return list.toString(); + } + + private void listRemove(long v) { + int sz = list.size(); + for (int i = 0; i < sz; i++) { + if (list.getQuick(i) == v) { + // shift remaining elements left + for (int j = i + 1; j < sz; j++) { + list.setQuick(j - 1, list.getQuick(j)); + } + list.setPos(sz - 1); + return; + } + } + } + + private void rehash() { + int newCapacity = capacity * 2; + free = capacity = newCapacity; + int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); + this.keys = new long[len]; + Arrays.fill(keys, noEntryKeyValue); + mask = len - 1; + int n = list.size(); + free -= n; + for (int i = 0; i < n; i++) { + long key = list.getQuick(i); + int keyIndex = keyIndex(key); + keys[keyIndex] = key; + } + } + + @Override + protected void erase(int index) { + keys[index] = noEntryKeyValue; + } + + @Override + protected void move(int from, int to) { + keys[to] = keys[from]; + erase(from); + } + +} diff --git a/core/src/main/java/module-info.java b/core/src/main/java/module-info.java index fa5bc48..45b319c 100644 --- a/core/src/main/java/module-info.java +++ b/core/src/main/java/module-info.java @@ -56,4 +56,7 @@ exports io.questdb.client.cairo.arr; exports io.questdb.client.cutlass.line.array; exports io.questdb.client.cutlass.line.udp; + exports io.questdb.client.cutlass.ilpv4.client; + exports io.questdb.client.cutlass.ilpv4.protocol; + exports io.questdb.client.cutlass.ilpv4.websocket; } From dee93bdeeef54b91c836f2f85c652afa27e2a802 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 20:27:34 +0000 Subject: [PATCH 002/230] tidy --- .../questdb/client/cutlass/http/client/WebSocketClient.java | 5 ++--- .../client/cutlass/http/client/WebSocketSendBuffer.java | 3 ++- .../client/cutlass/ilpv4/client/WebSocketResponse.java | 1 - .../client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 8f5ce83..cc64a63 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -29,8 +29,6 @@ import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import io.questdb.client.network.IOOperation; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.Socket; @@ -42,7 +40,8 @@ import io.questdb.client.std.Rnd; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; -import io.questdb.client.std.str.Utf8String; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; import java.util.Base64; diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 0eea869..20c43c1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; import io.questdb.client.cutlass.ilpv4.client.IlpBufferWriter; +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Numbers; import io.questdb.client.std.QuietCloseable; @@ -137,7 +138,7 @@ private void grow(long requiredCapacity) { .put(maxBufferSize) .put(']'); } - int newCapacity = (int) Math.min( + int newCapacity = Math.min( Numbers.ceilPow2((int) requiredCapacity), maxBufferSize ); diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java index 42e74af..2c29baa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java @@ -24,7 +24,6 @@ package io.questdb.client.cutlass.ilpv4.client; -import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import java.nio.charset.StandardCharsets; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java index 566e2f2..9996d7f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java @@ -29,8 +29,6 @@ import io.questdb.client.std.str.DirectUtf8Sequence; import io.questdb.client.std.str.Utf8Sequence; -import java.nio.charset.StandardCharsets; - /** * XXHash64 implementation for schema hashing in ILP v4 protocol. *

From ab2f5b58ebdaecf60eddad1209fd5ed17a772a9b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 21:11:38 +0000 Subject: [PATCH 003/230] move client test --- .../test/LineSenderBuilderWebSocketTest.java | 777 ++++++++++++++++++ 1 file changed, 777 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java diff --git a/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java new file mode 100644 index 0000000..736aec4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java @@ -0,0 +1,777 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Tests for WebSocket transport support in the Sender.builder() API. + * These tests verify the builder configuration and validation, + * not actual WebSocket connectivity (which requires a running server). + */ +public class LineSenderBuilderWebSocketTest extends AbstractTest { + + private static final String LOCALHOST = "localhost"; + + @Test + public void testAddressConfiguration() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000"); + Assert.assertNotNull(builder); + } + + @Test + public void testAddressEmpty_fails() { + assertThrows("address cannot be empty", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("")); + } + + @Test + public void testAddressEndsWithColon_fails() { + assertThrows("invalid address", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("foo:")); + } + + @Test + public void testAddressNull_fails() { + assertThrows("null", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address(null)); + } + + // ==================== Transport Selection Tests ==================== + + @Test + public void testAddressWithoutPort_usesDefaultPort9000() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST); + Assert.assertNotNull(builder); + } + + @Test + public void testAsyncModeCanBeSetMultipleTimes() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .asyncMode(false); + Assert.assertNotNull(builder); + } + + // ==================== Address Configuration Tests ==================== + + @Test + public void testAsyncModeDisabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false); + Assert.assertNotNull(builder); + } + + @Test + public void testAsyncModeEnabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true); + Assert.assertNotNull(builder); + } + + @Test + public void testAsyncModeWithAllOptions() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .autoFlushRows(500) + .autoFlushBytes(512 * 1024) + .autoFlushIntervalMillis(50) + .inFlightWindowSize(8) + .sendQueueCapacity(16); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushBytes() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(1024 * 1024); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushBytesDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(1024) + .autoFlushBytes(2048)); + } + + @Test + public void testAutoFlushBytesNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(-1)); + } + + @Test + public void testAutoFlushBytesZero() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(0); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushIntervalMillis() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(100); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushIntervalMillisDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(100) + .autoFlushIntervalMillis(200)); + } + + // ==================== TLS Configuration Tests ==================== + + @Test + public void testAutoFlushIntervalMillisNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(-1)); + } + + @Test + public void testAutoFlushIntervalMillisZero_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(0)); + } + + @Test + public void testAutoFlushRows() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(1000); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushRowsDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(100) + .autoFlushRows(200)); + } + + @Test + public void testAutoFlushRowsNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(-1)); + } + + // ==================== Async Mode Tests ==================== + + @Test + public void testAutoFlushRowsZero_disablesRowBasedAutoFlush() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(0); + Assert.assertNotNull(builder); + } + + @Test + public void testBufferCapacity() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(128 * 1024); + Assert.assertNotNull(builder); + } + + @Test + public void testBufferCapacityDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(1024) + .bufferCapacity(2048)); + } + + // ==================== Auto Flush Rows Tests ==================== + + @Test + public void testBufferCapacityNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(-1)); + } + + @Test + public void testBuilderWithWebSocketTransport() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET); + Assert.assertNotNull("Builder should be created for WebSocket transport", builder); + } + + @Test + public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000"), + "connect", "Failed" + ); + } + + @Test + public void testConnectionRefused() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":19999"), + "connect", "Failed" + ); + } + + // ==================== Auto Flush Bytes Tests ==================== + + @Test + public void testCustomTrustStore_butTlsNotEnabled_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .advancedTls().customTrustStore("/some/path", "password".toCharArray()) + .address(LOCALHOST), + "TLS was not enabled"); + } + + @Test + @Ignore("Disable auto flush may need different semantics for WebSocket") + public void testDisableAutoFlush_semantics() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush(); + Assert.assertNotNull(builder); + } + + @Test + public void testDnsResolutionFailure() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld:9000"), + "resolve", "connect", "Failed" + ); + } + + @Test + public void testDuplicateAddresses_fails() { + assertThrows("duplicated addresses", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9000")); + } + + // ==================== Auto Flush Interval Tests ==================== + + @Test + @Ignore("TCP authentication is not supported for WebSocket protocol") + public void testEnableAuth_notSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableAuth("keyId") + .authToken("token"), + "not supported for WebSocket"); + } + + @Test + public void testFullAsyncConfiguration() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .autoFlushRows(1000) + .autoFlushBytes(1024 * 1024) + .autoFlushIntervalMillis(100) + .inFlightWindowSize(16) + .sendQueueCapacity(32); + Assert.assertNotNull(builder); + } + + @Test + public void testFullAsyncConfigurationWithTls() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls() + .advancedTls().disableCertificateValidation() + .asyncMode(true) + .autoFlushRows(1000) + .autoFlushBytes(1024 * 1024) + .inFlightWindowSize(16) + .sendQueueCapacity(32); + Assert.assertNotNull(builder); + } + + @Test + @Ignore("HTTP path is HTTP-specific and may not apply to WebSocket") + public void testHttpPath_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpPath("/custom/path"); + Assert.assertNotNull(builder); + } + + // ==================== In-Flight Window Size Tests ==================== + + @Test + @Ignore("HTTP timeout is HTTP-specific and may not apply to WebSocket") + public void testHttpTimeout_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpTimeoutMillis(5000); + Assert.assertNotNull(builder); + } + + @Test + public void testHttpToken_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpToken("token"), + "not yet supported"); + } + + @Test + @Ignore("HTTP token authentication is not yet supported for WebSocket protocol") + public void testHttpToken_notYetSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpToken("token"), + "not yet supported"); + } + + @Test + public void testInFlightWindowSizeDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(8) + .inFlightWindowSize(16)); + } + + @Test + public void testInFlightWindowSizeNegative_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(-1)); + } + + // ==================== Send Queue Capacity Tests ==================== + + @Test + public void testInFlightWindowSizeZero_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(0)); + } + + @Test + public void testInFlightWindowSize_withAsyncMode() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(16); + Assert.assertNotNull(builder); + } + + @Test + public void testInFlightWindowSize_withoutAsyncMode_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(16), + "requires async mode"); + } + + @Test + public void testInvalidPort_fails() { + assertThrows("invalid port", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address(LOCALHOST + ":99999")); + } + + @Test + public void testInvalidSchema_fails() { + assertBadConfig("invalid::addr=localhost:9000;", "invalid schema [schema=invalid, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + } + + // ==================== Combined Async Configuration Tests ==================== + + @Test + public void testMalformedPortInAddress_fails() { + assertThrows("cannot parse a port from the address", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("foo:nonsense12334")); + } + + @Test + @Ignore("Max backoff is HTTP-specific and may not apply to WebSocket") + public void testMaxBackoff_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxBackoffMillis(1000); + Assert.assertNotNull(builder); + } + + // ==================== Config String Tests (ws:// and wss://) ==================== + + @Test + public void testMaxNameLength() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(256); + Assert.assertNotNull(builder); + } + + @Test + public void testMaxNameLengthDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(128) + .maxNameLength(256)); + } + + @Test + public void testMaxNameLengthTooSmall_fails() { + assertThrows("at least 16 bytes", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(10)); + } + + @Test + @Ignore("Min request throughput is HTTP-specific and may not apply to WebSocket") + public void testMinRequestThroughput_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .minRequestThroughput(10000); + Assert.assertNotNull(builder); + } + + @Test + public void testMultipleAddresses_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9001"), + "single address"); + } + + @Test + public void testNoAddress_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET), + "address not set"); + } + + @Test + public void testPortMismatch_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .port(9001), + "mismatch"); + } + + // ==================== Buffer Configuration Tests ==================== + + @Test + @Ignore("Protocol version is for ILP text protocol, WebSocket uses ILP v4 binary protocol") + public void testProtocolVersion_notApplicable() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .protocolVersion(Sender.PROTOCOL_VERSION_V2); + Assert.assertNotNull(builder); + } + + @Test + @Ignore("Retry timeout is HTTP-specific and may not apply to WebSocket") + public void testRetryTimeout_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .retryTimeoutMillis(5000); + Assert.assertNotNull(builder); + } + + @Test + public void testSendQueueCapacityDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(16) + .sendQueueCapacity(32)); + } + + // ==================== Unsupported Features (TCP Authentication) ==================== + + @Test + public void testSendQueueCapacityNegative_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(-1)); + } + + @Test + public void testSendQueueCapacityZero_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(0)); + } + + // ==================== Unsupported Features (HTTP Token Authentication) ==================== + + @Test + public void testSendQueueCapacity_withAsyncMode() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(32); + Assert.assertNotNull(builder); + } + + @Test + public void testSendQueueCapacity_withoutAsyncMode_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .sendQueueCapacity(32), + "requires async mode"); + } + + // ==================== Unsupported Features (Username/Password Authentication) ==================== + + @Test + public void testSyncModeDoesNotAllowInFlightWindowSize() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .inFlightWindowSize(16), + "requires async mode"); + } + + @Test + public void testSyncModeDoesNotAllowSendQueueCapacity() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .sendQueueCapacity(32), + "requires async mode"); + } + + // ==================== Unsupported Features (HTTP-specific options) ==================== + + @Test + public void testSyncModeIsDefault() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST); + Assert.assertNotNull(builder); + } + + @Test + public void testTcpAuth_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableAuth("keyId") + .authToken("5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), + "not supported for WebSocket"); + } + + @Test + public void testTlsDoubleSet_fails() { + assertThrows("already enabled", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .enableTls() + .enableTls()); + } + + @Test + public void testTlsEnabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls(); + Assert.assertNotNull(builder); + } + + @Test + public void testTlsValidationDisabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls() + .advancedTls().disableCertificateValidation(); + Assert.assertNotNull(builder); + } + + @Test + public void testTlsValidationDisabled_butTlsNotEnabled_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .advancedTls().disableCertificateValidation() + .address(LOCALHOST), + "TLS was not enabled"); + } + + // ==================== Unsupported Features (Protocol Version) ==================== + + @Test + public void testUsernamePassword_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpUsernamePassword("user", "pass"), + "not yet supported"); + } + + // ==================== Config String Unsupported Options ==================== + + @Test + @Ignore("Username/password authentication is not yet supported for WebSocket protocol") + public void testUsernamePassword_notYetSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpUsernamePassword("user", "pass"), + "not yet supported"); + } + + @Test + public void testWsConfigString() { + assertBadConfig("ws::addr=localhost:9000;", "connect", "Failed"); + } + + // ==================== Edge Cases ==================== + + @Test + public void testWsConfigString_missingAddr_fails() { + assertBadConfig("ws::addr=localhost;", "connect", "Failed"); + assertBadConfig("ws::foo=bar;", "addr is missing"); + } + + @Test + public void testWsConfigString_protocolAlreadyConfigured_fails() { + assertThrowsAny( + Sender.builder("ws::addr=localhost:9000;") + .enableTls(), + "TLS", "connect", "Failed" + ); + } + + @Test + public void testWsConfigString_uppercaseNotSupported() { + assertBadConfig("WS::addr=localhost:9000;", "invalid schema"); + } + + @Test + @Ignore("Token authentication in ws config string is not yet supported") + public void testWsConfigString_withToken_notYetSupported() { + assertBadConfig("ws::addr=localhost:9000;token=mytoken;", "not yet supported"); + } + + @Test + @Ignore("Username/password in ws config string is not yet supported") + public void testWsConfigString_withUsernamePassword_notYetSupported() { + assertBadConfig("ws::addr=localhost:9000;username=user;password=pass;", "not yet supported"); + } + + // ==================== Connection Tests ==================== + + @Test + public void testWssConfigString() { + assertBadConfig("wss::addr=localhost:9000;tls_verify=unsafe_off;", "connect", "Failed", "SSL"); + } + + @Test + public void testWssConfigString_uppercaseNotSupported() { + assertBadConfig("WSS::addr=localhost:9000;", "invalid schema"); + } + + // ==================== Sync vs Async Mode Tests ==================== + + @SuppressWarnings("resource") + private static void assertBadConfig(String config, String... anyOf) { + assertThrowsAny(() -> Sender.fromConfig(config), anyOf); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + Assert.fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void assertThrowsAny(Sender.LineSenderBuilder builder, String... anyOf) { + assertThrowsAny(builder::build, anyOf); + } + + private static void assertThrowsAny(Runnable action, String... anyOf) { + try { + action.run(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + for (String s : anyOf) { + if (msg.contains(s)) { + return; + } + } + Assert.fail("Expected message containing one of [" + String.join(", ", anyOf) + "] but got: " + msg); + } + } +} From 73ee623630ec47e2df403ef1f1a0f5030356493d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 21:54:40 +0000 Subject: [PATCH 004/230] more tidy --- .../client/BuildInformationHolder.java | 6 +- .../ilpv4/client/IlpV4WebSocketEncoder.java | 593 ++++++------- .../ilpv4/protocol/IlpV4GorillaDecoder.java | 251 ------ .../ilpv4/protocol/IlpV4GorillaEncoder.java | 53 +- .../ilpv4/protocol/IlpV4TimestampDecoder.java | 474 ----------- .../client/test/LineSenderBuilderTest.java | 799 ------------------ .../cutlass/line/LineSenderBuilderTest.java | 490 +++++++++++ .../line}/LineSenderBuilderWebSocketTest.java | 3 +- core/src/test/java/module-info.java | 1 + 9 files changed, 846 insertions(+), 1824 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java delete mode 100644 core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java rename core/src/test/java/io/questdb/client/test/{ => cutlass/line}/LineSenderBuilderWebSocketTest.java (99%) diff --git a/core/src/main/java/io/questdb/client/BuildInformationHolder.java b/core/src/main/java/io/questdb/client/BuildInformationHolder.java index e18f961..825a101 100644 --- a/core/src/main/java/io/questdb/client/BuildInformationHolder.java +++ b/core/src/main/java/io/questdb/client/BuildInformationHolder.java @@ -41,7 +41,7 @@ public BuildInformationHolder(Class clazz) { String swVersion; try { final Attributes manifestAttributes = getManifestAttributes(clazz); - swVersion = getAttr(manifestAttributes, "QuestDB-Client-Version", "[DEVELOPMENT]"); + swVersion = getAttr(manifestAttributes, "[DEVELOPMENT]"); } catch (IOException e) { swVersion = UNKNOWN; } @@ -57,8 +57,8 @@ public String getSwVersion() { return swVersion; } - private static String getAttr(final Attributes manifestAttributes, String attributeName, String defaultValue) { - final String value = manifestAttributes.getValue(attributeName); + private static String getAttr(final Attributes manifestAttributes, String defaultValue) { + final String value = manifestAttributes.getValue("QuestDB-Client-Version"); return value != null ? value : defaultValue; } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java index f1826f5..e98a1df 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java @@ -24,12 +24,9 @@ package io.questdb.client.cutlass.ilpv4.client; -import io.questdb.client.cutlass.ilpv4.protocol.*; - import io.questdb.client.cutlass.ilpv4.protocol.IlpV4ColumnDef; import io.questdb.client.cutlass.ilpv4.protocol.IlpV4GorillaEncoder; - -import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TimestampDecoder; +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TableBuffer; import io.questdb.client.std.QuietCloseable; import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; @@ -55,10 +52,18 @@ */ public class IlpV4WebSocketEncoder implements QuietCloseable { - private NativeBufferWriter ownedBuffer; - private IlpBufferWriter buffer; + /** + * Encoding flag for Gorilla-encoded timestamps. + */ + public static final byte ENCODING_GORILLA = 0x01; + /** + * Encoding flag for uncompressed timestamps. + */ + public static final byte ENCODING_UNCOMPRESSED = 0x00; private final IlpV4GorillaEncoder gorillaEncoder = new IlpV4GorillaEncoder(); + private IlpBufferWriter buffer; private byte flags; + private NativeBufferWriter ownedBuffer; public IlpV4WebSocketEncoder() { this.ownedBuffer = new NativeBufferWriter(); @@ -72,68 +77,14 @@ public IlpV4WebSocketEncoder(int bufferSize) { this.flags = 0; } - /** - * Returns the underlying buffer. - *

- * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, - * that buffer is returned. Otherwise, returns the internal buffer. - */ - public IlpBufferWriter getBuffer() { - return buffer; - } - - /** - * Sets an external buffer for encoding. - *

- * When set, the encoder writes directly to this buffer instead of its internal buffer. - * The caller is responsible for managing the external buffer's lifecycle. - *

- * Pass {@code null} to revert to using the internal buffer. - * - * @param externalBuffer the external buffer to use, or null to use internal buffer - */ - public void setBuffer(IlpBufferWriter externalBuffer) { - this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; - } - - /** - * Returns true if currently using an external buffer. - */ - public boolean isUsingExternalBuffer() { - return buffer != ownedBuffer; - } - - /** - * Resets the encoder for a new message. - *

- * If using an external buffer, this only resets the internal state (flags). - * The external buffer's reset is the caller's responsibility. - * If using the internal buffer, resets both the buffer and internal state. - */ - public void reset() { - if (!isUsingExternalBuffer()) { - buffer.reset(); - } - } - - /** - * Sets whether Gorilla timestamp encoding is enabled. - */ - public void setGorillaEnabled(boolean enabled) { - if (enabled) { - flags |= FLAG_GORILLA; - } else { - flags &= ~FLAG_GORILLA; + @Override + public void close() { + if (ownedBuffer != null) { + ownedBuffer.close(); + ownedBuffer = null; } } - /** - * Returns true if Gorilla encoding is enabled. - */ - public boolean isGorillaEnabled() { - return (flags & FLAG_GORILLA) != 0; - } - /** * Encodes a complete ILP v4 message from a table buffer. * @@ -164,11 +115,11 @@ public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { * This method sends only new symbols (delta) since the last confirmed watermark, * and uses global symbol IDs instead of per-column local indices. * - * @param tableBuffer the table buffer containing row data - * @param globalDict the global symbol dictionary - * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) - * @param batchMaxId the highest symbol ID used in this batch - * @param useSchemaRef whether to use schema reference mode + * @param tableBuffer the table buffer containing row data + * @param globalDict the global symbol dictionary + * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) + * @param batchMaxId the highest symbol ID used in this batch + * @param useSchemaRef whether to use schema reference mode * @return the number of bytes written */ public int encodeWithDeltaDict( @@ -214,14 +165,13 @@ public int encodeWithDeltaDict( } /** - * Sets the delta symbol dictionary flag. + * Returns the underlying buffer. + *

+ * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, + * that buffer is returned. Otherwise, returns the internal buffer. */ - public void setDeltaSymbolDictEnabled(boolean enabled) { - if (enabled) { - flags |= FLAG_DELTA_SYMBOL_DICT; - } else { - flags &= ~FLAG_DELTA_SYMBOL_DICT; - } + public IlpBufferWriter getBuffer() { + return buffer; } /** @@ -232,127 +182,92 @@ public boolean isDeltaSymbolDictEnabled() { } /** - * Writes the ILP v4 message header. - * - * @param tableCount number of tables in the message - * @param payloadLength payload length (can be 0 if patched later) + * Returns true if Gorilla encoding is enabled. */ - public void writeHeader(int tableCount, int payloadLength) { - // Magic "ILP4" - buffer.putByte((byte) 'I'); - buffer.putByte((byte) 'L'); - buffer.putByte((byte) 'P'); - buffer.putByte((byte) '4'); - - // Version - buffer.putByte(VERSION_1); - - // Flags - buffer.putByte(flags); - - // Table count (uint16, little-endian) - buffer.putShort((short) tableCount); - - // Payload length (uint32, little-endian) - buffer.putInt(payloadLength); + public boolean isGorillaEnabled() { + return (flags & FLAG_GORILLA) != 0; } /** - * Encodes a single table from the buffer. + * Returns true if currently using an external buffer. */ - private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - int rowCount = tableBuffer.getRowCount(); - - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - tableBuffer.getSchemaHash(), - columnDefs.length - ); - } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); - } + public boolean isUsingExternalBuffer() { + return buffer != ownedBuffer; + } - // Write each column's data - boolean useGorilla = isGorillaEnabled(); - for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; - encodeColumn(col, colDef, rowCount, useGorilla); + /** + * Resets the encoder for a new message. + *

+ * If using an external buffer, this only resets the internal state (flags). + * The external buffer's reset is the caller's responsibility. + * If using the internal buffer, resets both the buffer and internal state. + */ + public void reset() { + if (!isUsingExternalBuffer()) { + buffer.reset(); } } /** - * Encodes a single table from the buffer using global symbol IDs. - * This is used with delta dictionary encoding. + * Sets an external buffer for encoding. + *

+ * When set, the encoder writes directly to this buffer instead of its internal buffer. + * The caller is responsible for managing the external buffer's lifecycle. + *

+ * Pass {@code null} to revert to using the internal buffer. + * + * @param externalBuffer the external buffer to use, or null to use internal buffer */ - private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - int rowCount = tableBuffer.getRowCount(); + public void setBuffer(IlpBufferWriter externalBuffer) { + this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; + } - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - tableBuffer.getSchemaHash(), - columnDefs.length - ); + /** + * Sets the delta symbol dictionary flag. + */ + public void setDeltaSymbolDictEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_DELTA_SYMBOL_DICT; } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); - } - - // Write each column's data - boolean useGorilla = isGorillaEnabled(); - for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; - encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + flags &= ~FLAG_DELTA_SYMBOL_DICT; } } /** - * Writes a table header with full schema. + * Sets whether Gorilla timestamp encoding is enabled. */ - private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { - // Table name - buffer.putString(tableName); - - // Row count (varint) - buffer.putVarint(rowCount); - - // Column count (varint) - buffer.putVarint(columns.length); - - // Schema mode: full schema (0x00) - buffer.putByte(SCHEMA_MODE_FULL); - - // Column definitions (name + type for each) - for (IlpV4ColumnDef col : columns) { - buffer.putString(col.getName()); - buffer.putByte(col.getWireTypeCode()); + public void setGorillaEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_GORILLA; + } else { + flags &= ~FLAG_GORILLA; } } /** - * Writes a table header with schema reference. + * Writes the ILP v4 message header. + * + * @param tableCount number of tables in the message + * @param payloadLength payload length (can be 0 if patched later) */ - private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { - // Table name - buffer.putString(tableName); + public void writeHeader(int tableCount, int payloadLength) { + // Magic "ILP4" + buffer.putByte((byte) 'I'); + buffer.putByte((byte) 'L'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '4'); - // Row count (varint) - buffer.putVarint(rowCount); + // Version + buffer.putByte(VERSION_1); - // Column count (varint) - buffer.putVarint(columnCount); + // Flags + buffer.putByte(flags); - // Schema mode: reference (0x01) - buffer.putByte(SCHEMA_MODE_REFERENCE); + // Table count (uint16, little-endian) + buffer.putShort((short) tableCount); - // Schema hash (8 bytes) - buffer.putLong(schemaHash); + // Payload length (uint32, little-endian) + buffer.putInt(payloadLength); } /** @@ -513,20 +428,61 @@ private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, Il } /** - * Writes a null bitmap from bit-packed long array. + * Encodes a single table from the buffer. */ - private void writeNullBitmapPacked(long[] nullsPacked, int count) { - int bitmapSize = (count + 7) / 8; - - for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { - int longIndex = byteIdx >>> 3; - int byteInLong = byteIdx & 7; - byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); - buffer.putByte(b); - } - } + private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); - /** + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumn(col, colDef, rowCount, useGorilla); + } + } + + /** + * Encodes a single table from the buffer using global symbol IDs. + * This is used with delta dictionary encoding. + */ + private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + } + } + + /** * Writes boolean column data (bit-packed). */ private void writeBooleanColumn(boolean[] values, int count) { @@ -550,21 +506,58 @@ private void writeByteColumn(byte[] values, int count) { } } - private void writeShortColumn(short[] values, int count) { + private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { + buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putShort(values[i]); + buffer.putLongBE(high[i]); + buffer.putLongBE(low[i]); } } - private void writeIntColumn(int[] values, int count) { + private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { + buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putInt(values[i]); + buffer.putLongBE(hh[i]); + buffer.putLongBE(hl[i]); + buffer.putLongBE(lh[i]); + buffer.putLongBE(ll[i]); } } - private void writeLongColumn(long[] values, int count) { + private void writeDecimal64Column(byte scale, long[] values, int count) { + buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLong(values[i]); + buffer.putLongBE(values[i]); + } + } + + private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putDouble(data[dataIdx++]); + } + } + } + + private void writeDoubleColumn(double[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putDouble(values[i]); } } @@ -574,42 +567,67 @@ private void writeFloatColumn(float[] values, int count) { } } - private void writeDoubleColumn(double[] values, int count) { + private void writeIntColumn(int[] values, int count) { for (int i = 0; i < count; i++) { - buffer.putDouble(values[i]); + buffer.putInt(values[i]); + } + } + + private void writeLong256Column(long[] values, int count) { + // Flat array: 4 longs per value, little-endian (least significant first) + // values layout: [long0, long1, long2, long3] per row + for (int i = 0; i < count * 4; i++) { + buffer.putLong(values[i]); + } + } + + private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putLong(data[dataIdx++]); + } + } + } + + private void writeLongColumn(long[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putLong(values[i]); } } /** - * Writes a timestamp column with optional Gorilla compression. - *

- * When Gorilla encoding is enabled and applicable (3+ timestamps with - * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. - * Otherwise, falls back to uncompressed encoding. + * Writes a null bitmap from bit-packed long array. */ - private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { - if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { - // Write Gorilla encoding flag - buffer.putByte(IlpV4TimestampDecoder.ENCODING_GORILLA); + private void writeNullBitmapPacked(long[] nullsPacked, int count) { + int bitmapSize = (count + 7) / 8; - // Calculate size needed and ensure buffer has capacity - int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); - buffer.ensureCapacity(encodedSize); + for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { + int longIndex = byteIdx >>> 3; + int byteInLong = byteIdx & 7; + byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); + buffer.putByte(b); + } + } - // Encode timestamps to buffer - int bytesWritten = gorillaEncoder.encodeTimestamps( - buffer.getBufferPtr() + buffer.getPosition(), - buffer.getCapacity() - buffer.getPosition(), - values, - count - ); - buffer.skip(bytesWritten); - } else { - // Write uncompressed - if (useGorilla) { - buffer.putByte(IlpV4TimestampDecoder.ENCODING_UNCOMPRESSED); - } - writeLongColumn(values, count); + private void writeShortColumn(short[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putShort(values[i]); } } @@ -691,100 +709,87 @@ private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, i } } - private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { - // Little-endian: lo first, then hi - for (int i = 0; i < count; i++) { - buffer.putLong(lowBits[i]); - buffer.putLong(highBits[i]); - } - } - - private void writeLong256Column(long[] values, int count) { - // Flat array: 4 longs per value, little-endian (least significant first) - // values layout: [long0, long1, long2, long3] per row - for (int i = 0; i < count * 4; i++) { - buffer.putLong(values[i]); - } - } + /** + * Writes a table header with full schema. + */ + private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { + // Table name + buffer.putString(tableName); - private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - double[] data = col.getDoubleArrayData(); + // Row count (varint) + buffer.putVarint(rowCount); - int shapeIdx = 0; - int dataIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - buffer.putByte((byte) nDims); + // Column count (varint) + buffer.putVarint(columns.length); - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - int dimLen = shapes[shapeIdx++]; - buffer.putInt(dimLen); - elemCount *= dimLen; - } + // Schema mode: full schema (0x00) + buffer.putByte(SCHEMA_MODE_FULL); - for (int e = 0; e < elemCount; e++) { - buffer.putDouble(data[dataIdx++]); - } + // Column definitions (name + type for each) + for (IlpV4ColumnDef col : columns) { + buffer.putString(col.getName()); + buffer.putByte(col.getWireTypeCode()); } } - private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - long[] data = col.getLongArrayData(); + /** + * Writes a table header with schema reference. + */ + private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { + // Table name + buffer.putString(tableName); - int shapeIdx = 0; - int dataIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - buffer.putByte((byte) nDims); + // Row count (varint) + buffer.putVarint(rowCount); - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - int dimLen = shapes[shapeIdx++]; - buffer.putInt(dimLen); - elemCount *= dimLen; - } + // Column count (varint) + buffer.putVarint(columnCount); - for (int e = 0; e < elemCount; e++) { - buffer.putLong(data[dataIdx++]); - } - } - } + // Schema mode: reference (0x01) + buffer.putByte(SCHEMA_MODE_REFERENCE); - private void writeDecimal64Column(byte scale, long[] values, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - buffer.putLongBE(values[i]); - } + // Schema hash (8 bytes) + buffer.putLong(schemaHash); } - private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - buffer.putLongBE(high[i]); - buffer.putLongBE(low[i]); - } - } + /** + * Writes a timestamp column with optional Gorilla compression. + *

+ * When Gorilla encoding is enabled and applicable (3+ timestamps with + * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. + * Otherwise, falls back to uncompressed encoding. + */ + private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { + if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { + // Write Gorilla encoding flag + buffer.putByte(ENCODING_GORILLA); - private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - buffer.putLongBE(hh[i]); - buffer.putLongBE(hl[i]); - buffer.putLongBE(lh[i]); - buffer.putLongBE(ll[i]); + // Calculate size needed and ensure buffer has capacity + int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); + buffer.ensureCapacity(encodedSize); + + // Encode timestamps to buffer + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + values, + count + ); + buffer.skip(bytesWritten); + } else { + // Write uncompressed + if (useGorilla) { + buffer.putByte(ENCODING_UNCOMPRESSED); + } + writeLongColumn(values, count); } } - @Override - public void close() { - if (ownedBuffer != null) { - ownedBuffer.close(); - ownedBuffer = null; + private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { + // Little-endian: lo first, then hi + for (int i = 0; i < count; i++) { + buffer.putLong(lowBits[i]); + buffer.putLong(highBits[i]); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java deleted file mode 100644 index 2471ad4..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java +++ /dev/null @@ -1,251 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.ilpv4.protocol; - -/** - * Gorilla delta-of-delta decoder for timestamps in ILP v4 format. - *

- * Gorilla encoding uses delta-of-delta compression where: - * 

- * D = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
- *
- * if D == 0:              write '0'              (1 bit)
- * elif D in [-63, 64]:    write '10' + 7-bit     (9 bits)
- * elif D in [-255, 256]:  write '110' + 9-bit    (12 bits)
- * elif D in [-2047, 2048]: write '1110' + 12-bit (16 bits)
- * else:                   write '1111' + 32-bit  (36 bits)
- *
- *

- * The decoder reads bit-packed delta-of-delta values and reconstructs - * the original timestamp sequence. - */ -public class IlpV4GorillaDecoder { - - // Bucket boundaries (two's complement signed ranges) - private static final int BUCKET_7BIT_MIN = -63; - private static final int BUCKET_7BIT_MAX = 64; - private static final int BUCKET_9BIT_MIN = -255; - private static final int BUCKET_9BIT_MAX = 256; - private static final int BUCKET_12BIT_MIN = -2047; - private static final int BUCKET_12BIT_MAX = 2048; - - private final IlpV4BitReader bitReader; - - // State for decoding - private long prevTimestamp; - private long prevDelta; - - /** - * Creates a new Gorilla decoder. - */ - public IlpV4GorillaDecoder() { - this.bitReader = new IlpV4BitReader(); - } - - /** - * Creates a decoder using an existing bit reader. - * - * @param bitReader the bit reader to use - */ - public IlpV4GorillaDecoder(IlpV4BitReader bitReader) { - this.bitReader = bitReader; - } - - /** - * Resets the decoder with the first two timestamps. - *

- * The first two timestamps are always stored uncompressed and are used - * to establish the initial delta for subsequent compression. - * - * @param firstTimestamp the first timestamp in the sequence - * @param secondTimestamp the second timestamp in the sequence - */ - public void reset(long firstTimestamp, long secondTimestamp) { - this.prevTimestamp = secondTimestamp; - this.prevDelta = secondTimestamp - firstTimestamp; - } - - /** - * Resets the bit reader for reading encoded delta-of-deltas. - * - * @param address the address of the encoded data - * @param length the length of the encoded data in bytes - */ - public void resetReader(long address, long length) { - bitReader.reset(address, length); - } - - /** - * Decodes the next timestamp from the bit stream. - *

- * The encoding format is: - *

    - *
  • '0' = delta-of-delta is 0 (1 bit)
    • - *
  • '10' + 7-bit signed = delta-of-delta in [-63, 64] (9 bits)
    • - *
  • '110' + 9-bit signed = delta-of-delta in [-255, 256] (12 bits)
    • - *
  • '1110' + 12-bit signed = delta-of-delta in [-2047, 2048] (16 bits)
    • - *
  • '1111' + 32-bit signed = any other delta-of-delta (36 bits)
    • - *
- * - * @return the decoded timestamp - */ - public long decodeNext() { - long deltaOfDelta = decodeDoD(); - long delta = prevDelta + deltaOfDelta; - long timestamp = prevTimestamp + delta; - - prevDelta = delta; - prevTimestamp = timestamp; - - return timestamp; - } - - /** - * Decodes a delta-of-delta value from the bit stream. - * - * @return the delta-of-delta value - */ - private long decodeDoD() { - int bit = bitReader.readBit(); - - if (bit == 0) { - // '0' = DoD is 0 - return 0; - } - - // bit == 1, check next bit - bit = bitReader.readBit(); - if (bit == 0) { - // '10' = 7-bit signed value - return bitReader.readSigned(7); - } - - // '11', check next bit - bit = bitReader.readBit(); - if (bit == 0) { - // '110' = 9-bit signed value - return bitReader.readSigned(9); - } - - // '111', check next bit - bit = bitReader.readBit(); - if (bit == 0) { - // '1110' = 12-bit signed value - return bitReader.readSigned(12); - } - - // '1111' = 32-bit signed value - return bitReader.readSigned(32); - } - - /** - * Returns whether there are more bits available in the reader. - * - * @return true if more bits available - */ - public boolean hasMoreBits() { - return bitReader.hasMoreBits(); - } - - /** - * Returns the number of bits remaining. - * - * @return available bits - */ - public long getAvailableBits() { - return bitReader.getAvailableBits(); - } - - /** - * Returns the current bit position (bits read since reset). - * - * @return bits read - */ - public long getBitPosition() { - return bitReader.getBitPosition(); - } - - /** - * Gets the previous timestamp (for debugging/testing). - * - * @return the last decoded timestamp - */ - public long getPrevTimestamp() { - return prevTimestamp; - } - - /** - * Gets the previous delta (for debugging/testing). - * - * @return the last computed delta - */ - public long getPrevDelta() { - return prevDelta; - } - - // ==================== Static Encoding Methods (for testing) ==================== - - /** - * Determines which bucket a delta-of-delta value falls into. - * - * @param deltaOfDelta the delta-of-delta value - * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit) - */ - public static int getBucket(long deltaOfDelta) { - if (deltaOfDelta == 0) { - return 0; // 1-bit - } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) { - return 1; // 9-bit (2 prefix + 7 value) - } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) { - return 2; // 12-bit (3 prefix + 9 value) - } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) { - return 3; // 16-bit (4 prefix + 12 value) - } else { - return 4; // 36-bit (4 prefix + 32 value) - } - } - - /** - * Returns the number of bits required to encode a delta-of-delta value. - * - * @param deltaOfDelta the delta-of-delta value - * @return bits required - */ - public static int getBitsRequired(long deltaOfDelta) { - int bucket = getBucket(deltaOfDelta); - switch (bucket) { - case 0: - return 1; - case 1: - return 9; - case 2: - return 12; - case 3: - return 16; - default: - return 36; - } - } -} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java index e8f0f47..84e4334 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java @@ -46,6 +46,13 @@ */ public class IlpV4GorillaEncoder { + private static final int BUCKET_12BIT_MAX = 2048; + private static final int BUCKET_12BIT_MIN = -2047; + private static final int BUCKET_7BIT_MAX = 64; + // Bucket boundaries (two's complement signed ranges) + private static final int BUCKET_7BIT_MIN = -63; + private static final int BUCKET_9BIT_MAX = 256; + private static final int BUCKET_9BIT_MIN = -255; private final IlpV4BitWriter bitWriter = new IlpV4BitWriter(); /** @@ -54,6 +61,48 @@ public class IlpV4GorillaEncoder { public IlpV4GorillaEncoder() { } + /** + * Returns the number of bits required to encode a delta-of-delta value. + * + * @param deltaOfDelta the delta-of-delta value + * @return bits required + */ + public static int getBitsRequired(long deltaOfDelta) { + int bucket = getBucket(deltaOfDelta); + switch (bucket) { + case 0: + return 1; + case 1: + return 9; + case 2: + return 12; + case 3: + return 16; + default: + return 36; + } + } + + /** + * Determines which bucket a delta-of-delta value falls into. + * + * @param deltaOfDelta the delta-of-delta value + * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit) + */ + public static int getBucket(long deltaOfDelta) { + if (deltaOfDelta == 0) { + return 0; // 1-bit + } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) { + return 1; // 9-bit (2 prefix + 7 value) + } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) { + return 2; // 12-bit (3 prefix + 9 value) + } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) { + return 3; // 16-bit (4 prefix + 12 value) + } else { + return 4; // 36-bit (4 prefix + 32 value) + } + } + /** * Encodes a delta-of-delta value using bucket selection. *

@@ -69,7 +118,7 @@ public IlpV4GorillaEncoder() { * @param deltaOfDelta the delta-of-delta value to encode */ public void encodeDoD(long deltaOfDelta) { - int bucket = IlpV4GorillaDecoder.getBucket(deltaOfDelta); + int bucket = getBucket(deltaOfDelta); switch (bucket) { case 0: // DoD == 0 bitWriter.writeBit(0); @@ -221,7 +270,7 @@ public static int calculateEncodedSize(long[] timestamps, int count) { long delta = timestamps[i] - prevTimestamp; long deltaOfDelta = delta - prevDelta; - totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); + totalBits += getBitsRequired(deltaOfDelta); prevDelta = delta; prevTimestamp = timestamps[i]; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java deleted file mode 100644 index 3b4fffa..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java +++ /dev/null @@ -1,474 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.ilpv4.protocol; - -import io.questdb.client.std.Unsafe; - -/** - * Decoder for TIMESTAMP columns in ILP v4 format. - *

- * Supports two encoding modes: - *

    - *
  • Uncompressed (0x00): array of int64 values
    • - *
  • Gorilla (0x01): delta-of-delta compressed
    • - *
- *

- * Gorilla format: - * 

- * [Null bitmap if nullable]
- * First timestamp: int64 (8 bytes, little-endian)
- * Second timestamp: int64 (8 bytes, little-endian)
- * Remaining timestamps: bit-packed delta-of-delta
- *
- */ -public final class IlpV4TimestampDecoder { - - /** - * Encoding flag for uncompressed timestamps. - */ - public static final byte ENCODING_UNCOMPRESSED = 0x00; - - /** - * Encoding flag for Gorilla-encoded timestamps. - */ - public static final byte ENCODING_GORILLA = 0x01; - - public static final IlpV4TimestampDecoder INSTANCE = new IlpV4TimestampDecoder(); - - private final IlpV4GorillaDecoder gorillaDecoder = new IlpV4GorillaDecoder(); - - private IlpV4TimestampDecoder() { - } - - /** - * Decodes timestamp column data from native memory. - * - * @param sourceAddress source address in native memory - * @param sourceLength length of source data in bytes - * @param rowCount number of rows to decode - * @param nullable whether the column is nullable - * @param sink sink to receive decoded values - * @return number of bytes consumed - */ - public int decode(long sourceAddress, int sourceLength, int rowCount, boolean nullable, ColumnSink sink) { - if (rowCount == 0) { - return 0; - } - - int offset = 0; - - // Parse null bitmap if nullable - long nullBitmapAddress = 0; - if (nullable) { - int nullBitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); - if (offset + nullBitmapSize > sourceLength) { - throw new IllegalArgumentException("insufficient data for null bitmap"); - } - nullBitmapAddress = sourceAddress + offset; - offset += nullBitmapSize; - } - - // Read encoding flag - if (offset + 1 > sourceLength) { - throw new IllegalArgumentException("insufficient data for encoding flag"); - } - byte encoding = Unsafe.getUnsafe().getByte(sourceAddress + offset); - offset++; - - if (encoding == ENCODING_UNCOMPRESSED) { - offset = decodeUncompressed(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); - } else if (encoding == ENCODING_GORILLA) { - offset = decodeGorilla(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); - } else { - throw new IllegalArgumentException("unknown timestamp encoding: " + encoding); - } - - return offset; - } - - private int decodeUncompressed(long sourceAddress, int sourceLength, int offset, int rowCount, - boolean nullable, long nullBitmapAddress, ColumnSink sink) { - // Count nulls to determine actual value count - int nullCount = 0; - if (nullable) { - nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); - } - int valueCount = rowCount - nullCount; - - // Uncompressed: valueCount * 8 bytes - int valuesSize = valueCount * 8; - if (offset + valuesSize > sourceLength) { - throw new IllegalArgumentException("insufficient data for uncompressed timestamps"); - } - - long valuesAddress = sourceAddress + offset; - int valueOffset = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - long value = Unsafe.getUnsafe().getLong(valuesAddress + (long) valueOffset * 8); - sink.putLong(i, value); - valueOffset++; - } - } - - return offset + valuesSize; - } - - private int decodeGorilla(long sourceAddress, int sourceLength, int offset, int rowCount, - boolean nullable, long nullBitmapAddress, ColumnSink sink) { - // Count nulls to determine actual value count - int nullCount = 0; - if (nullable) { - nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); - } - int valueCount = rowCount - nullCount; - - if (valueCount == 0) { - // All nulls - for (int i = 0; i < rowCount; i++) { - sink.putNull(i); - } - return offset; - } - - // First timestamp: 8 bytes - if (offset + 8 > sourceLength) { - throw new IllegalArgumentException("insufficient data for first timestamp"); - } - long firstTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); - offset += 8; - - if (valueCount == 1) { - // Only one non-null value, output it at the appropriate row position - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - sink.putLong(i, firstTimestamp); - } - } - return offset; - } - - // Second timestamp: 8 bytes - if (offset + 8 > sourceLength) { - throw new IllegalArgumentException("insufficient data for second timestamp"); - } - long secondTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); - offset += 8; - - if (valueCount == 2) { - // Two non-null values - int valueIdx = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - sink.putLong(i, valueIdx == 0 ? firstTimestamp : secondTimestamp); - valueIdx++; - } - } - return offset; - } - - // Remaining timestamps: bit-packed delta-of-delta - // Reset the Gorilla decoder with the initial state - gorillaDecoder.reset(firstTimestamp, secondTimestamp); - - // Calculate remaining bytes for bit data - int remainingBytes = sourceLength - offset; - gorillaDecoder.resetReader(sourceAddress + offset, remainingBytes); - - // Decode timestamps and distribute to rows - int valueIdx = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - long timestamp; - if (valueIdx == 0) { - timestamp = firstTimestamp; - } else if (valueIdx == 1) { - timestamp = secondTimestamp; - } else { - timestamp = gorillaDecoder.decodeNext(); - } - sink.putLong(i, timestamp); - valueIdx++; - } - } - - // Calculate how many bytes were consumed - // The bit reader has consumed some bits; round up to bytes - long bitsRead = gorillaDecoder.getAvailableBits(); - long totalBits = remainingBytes * 8L; - long bitsConsumed = totalBits - bitsRead; - int bytesConsumed = (int) ((bitsConsumed + 7) / 8); - - return offset + bytesConsumed; - } - - /** - * Returns the expected size for decoding. - * - * @param rowCount number of rows - * @param nullable whether the column is nullable - * @return expected size in bytes - */ - public int expectedSize(int rowCount, boolean nullable) { - // Minimum size: just encoding flag + uncompressed timestamps - int size = 1; // encoding flag - if (nullable) { - size += IlpV4NullBitmap.sizeInBytes(rowCount); - } - size += rowCount * 8; // worst case: uncompressed - return size; - } - - // ==================== Static Encoding Methods (for testing) ==================== - - /** - * Encodes timestamps in uncompressed format to direct memory. - * Only non-null values are written. - * - * @param destAddress destination address - * @param timestamps timestamp values - * @param nulls null flags (can be null if not nullable) - * @return address after encoded data - */ - public static long encodeUncompressed(long destAddress, long[] timestamps, boolean[] nulls) { - int rowCount = timestamps.length; - boolean nullable = nulls != null; - long pos = destAddress; - - // Write null bitmap if nullable - if (nullable) { - int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); - IlpV4NullBitmap.fillNoneNull(pos, rowCount); - for (int i = 0; i < rowCount; i++) { - if (nulls[i]) { - IlpV4NullBitmap.setNull(pos, i); - } - } - pos += bitmapSize; - } - - // Write encoding flag - Unsafe.getUnsafe().putByte(pos++, ENCODING_UNCOMPRESSED); - - // Write only non-null timestamps - for (int i = 0; i < rowCount; i++) { - if (nullable && nulls[i]) continue; - Unsafe.getUnsafe().putLong(pos, timestamps[i]); - pos += 8; - } - - return pos; - } - - /** - * Encodes timestamps in Gorilla format to direct memory. - * Only non-null values are encoded. - * - * @param destAddress destination address - * @param timestamps timestamp values - * @param nulls null flags (can be null if not nullable) - * @return address after encoded data - */ - public static long encodeGorilla(long destAddress, long[] timestamps, boolean[] nulls) { - int rowCount = timestamps.length; - boolean nullable = nulls != null; - long pos = destAddress; - - // Write null bitmap if nullable - if (nullable) { - int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); - IlpV4NullBitmap.fillNoneNull(pos, rowCount); - for (int i = 0; i < rowCount; i++) { - if (nulls[i]) { - IlpV4NullBitmap.setNull(pos, i); - } - } - pos += bitmapSize; - } - - // Count non-null values - int valueCount = 0; - for (int i = 0; i < rowCount; i++) { - if (!nullable || !nulls[i]) valueCount++; - } - - // Write encoding flag - Unsafe.getUnsafe().putByte(pos++, ENCODING_GORILLA); - - if (valueCount == 0) { - return pos; - } - - // Build array of non-null values - long[] nonNullValues = new long[valueCount]; - int idx = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && nulls[i]) continue; - nonNullValues[idx++] = timestamps[i]; - } - - // Write first timestamp - Unsafe.getUnsafe().putLong(pos, nonNullValues[0]); - pos += 8; - - if (valueCount == 1) { - return pos; - } - - // Write second timestamp - Unsafe.getUnsafe().putLong(pos, nonNullValues[1]); - pos += 8; - - if (valueCount == 2) { - return pos; - } - - // Encode remaining timestamps using Gorilla - IlpV4BitWriter bitWriter = new IlpV4BitWriter(); - bitWriter.reset(pos, 1024 * 1024); // 1MB max for bit data - - long prevTimestamp = nonNullValues[1]; - long prevDelta = nonNullValues[1] - nonNullValues[0]; - - for (int i = 2; i < valueCount; i++) { - long delta = nonNullValues[i] - prevTimestamp; - long deltaOfDelta = delta - prevDelta; - - encodeDoD(bitWriter, deltaOfDelta); - - prevDelta = delta; - prevTimestamp = nonNullValues[i]; - } - - // Flush remaining bits - int bytesWritten = bitWriter.finish(); - pos += bytesWritten; - - return pos; - } - - /** - * Encodes a delta-of-delta value to the bit writer. - *

- * Prefix patterns are written LSB-first to match the decoder's read order: - * - '0' -> write bit 0 - * - '10' -> write bit 1, then bit 0 (0b01 as 2-bit value) - * - '110' -> write bit 1, bit 1, bit 0 (0b011 as 3-bit value) - * - '1110' -> write bit 1, bit 1, bit 1, bit 0 (0b0111 as 4-bit value) - * - '1111' -> write bit 1, bit 1, bit 1, bit 1 (0b1111 as 4-bit value) - */ - private static void encodeDoD(IlpV4BitWriter writer, long deltaOfDelta) { - if (deltaOfDelta == 0) { - // '0' = DoD is 0 - writer.writeBit(0); - } else if (deltaOfDelta >= -63 && deltaOfDelta <= 64) { - // '10' prefix: first bit read=1, second bit read=0 -> write as 0b01 (LSB-first) - writer.writeBits(0b01, 2); - writer.writeSigned(deltaOfDelta, 7); - } else if (deltaOfDelta >= -255 && deltaOfDelta <= 256) { - // '110' prefix: bits read as 1,1,0 -> write as 0b011 (LSB-first) - writer.writeBits(0b011, 3); - writer.writeSigned(deltaOfDelta, 9); - } else if (deltaOfDelta >= -2047 && deltaOfDelta <= 2048) { - // '1110' prefix: bits read as 1,1,1,0 -> write as 0b0111 (LSB-first) - writer.writeBits(0b0111, 4); - writer.writeSigned(deltaOfDelta, 12); - } else { - // '1111' prefix: bits read as 1,1,1,1 -> write as 0b1111 (LSB-first) - writer.writeBits(0b1111, 4); - writer.writeSigned(deltaOfDelta, 32); - } - } - - /** - * Calculates the encoded size in bytes for Gorilla-encoded timestamps. - * - * @param timestamps timestamp values - * @param nullable whether column is nullable - * @return encoded size in bytes - */ - public static int calculateGorillaSize(long[] timestamps, boolean nullable) { - int rowCount = timestamps.length; - int size = 0; - - if (nullable) { - size += IlpV4NullBitmap.sizeInBytes(rowCount); - } - - size += 1; // encoding flag - - if (rowCount == 0) { - return size; - } - - size += 8; // first timestamp - - if (rowCount == 1) { - return size; - } - - size += 8; // second timestamp - - if (rowCount == 2) { - return size; - } - - // Calculate bits for delta-of-delta encoding - long prevTimestamp = timestamps[1]; - long prevDelta = timestamps[1] - timestamps[0]; - int totalBits = 0; - - for (int i = 2; i < rowCount; i++) { - long delta = timestamps[i] - prevTimestamp; - long deltaOfDelta = delta - prevDelta; - - totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); - - prevDelta = delta; - prevTimestamp = timestamps[i]; - } - - // Round up to bytes - size += (totalBits + 7) / 8; - - return size; - } - - /** - * Sink interface for receiving decoded column values. - */ - public interface ColumnSink { - void putLong(int rowIndex, long value); - void putNull(int rowIndex); - } -} diff --git a/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java deleted file mode 100644 index fe72c01..0000000 --- a/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java +++ /dev/null @@ -1,799 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; -import static org.junit.Assert.fail; - -/** - * Unit tests for LineSenderBuilder that don't require a running QuestDB instance. - * Tests that require an actual QuestDB connection have been moved to integration tests. - */ -public class LineSenderBuilderTest { - private static final String AUTH_TOKEN_KEY1 = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; - private static final String LOCALHOST = "localhost"; - private static final char[] TRUSTSTORE_PASSWORD = "questdb".toCharArray(); - private static final String TRUSTSTORE_PATH = "/keystore/server.keystore"; - - @Test - public void testAddressDoubleSet_firstAddressThenAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST); - try { - builder.address("127.0.0.1"); - builder.build(); - fail("should not allow double host set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testAddressEmpty() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address(""); - fail("empty address should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "address cannot be empty"); - } - }); - } - - @Test - public void testAddressEndsWithColon() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address("foo:"); - fail("should fail when address ends with colon"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "invalid address"); - } - }); - } - - @Test - public void testAddressNull() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address(null); - fail("null address should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "null"); - } - }); - } - - @Test - public void testAuthDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken(AUTH_TOKEN_KEY1); - try { - builder.enableAuth("bar"); - fail("should not allow double auth set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testAuthTooSmallBuffer() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":9001") - .bufferCapacity(1); - builder.build(); - fail("tiny buffer should NOT be allowed as it wont fit auth challenge"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimalCapacity"); - TestUtils.assertContains(e.getMessage(), "requestedCapacity"); - } - }); - } - - @Test - public void testAuthWithBadToken() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder.AuthBuilder builder = Sender.builder(Sender.Transport.TCP).enableAuth("foo"); - try { - builder.authToken("bar token"); - fail("bad token should not be imported"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not import token"); - } - }); - } - - @Test - public void testAutoFlushIntervalMustBePositive() { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(0).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval cannot be negative [autoFlushIntervalMillis=0]"); - } - - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(-1).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval cannot be negative [autoFlushIntervalMillis=-1]"); - } - } - - @Test - public void testAutoFlushIntervalNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushIntervalMillis(1).build(); - fail("auto flush interval should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval is not supported for TCP protocol"); - } - }); - } - - @Test - public void testAutoFlushInterval_afterAutoFlushDisabled() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).disableAutoFlush().autoFlushIntervalMillis(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "cannot set auto flush interval when interval based auto-flush is already disabled"); - } - }); - } - - @Test - public void testAutoFlushInterval_doubleConfiguration() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(1).autoFlushIntervalMillis(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval was already configured [autoFlushIntervalMillis=1]"); - } - }); - } - - @Test - public void testAutoFlushRowsCannotBeNegative() { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushRows(-1).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows cannot be negative [autoFlushRows=-1]"); - } - } - - @Test - public void testAutoFlushRowsNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushRows(1).build(); - fail("auto flush rows should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows is not supported for TCP protocol"); - } - }); - } - - @Test - public void testAutoFlushRows_doubleConfiguration() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).autoFlushRows(1).autoFlushRows(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows was already configured [autoFlushRows=1]"); - } - }); - } - - @Test - public void testBufferSizeDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).bufferCapacity(1024); - try { - builder.bufferCapacity(1024); - fail("should not allow double buffer capacity set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testConfStringValidation() throws Exception { - assertMemoryLeak(() -> { - assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("http::addr=localhost:-1;", "invalid port [port=-1]"); - assertConfStrError("http::auto_flush=on;", "addr is missing"); - assertConfStrError("http::addr=localhost;tls_roots=/some/path;", "tls_roots was configured, but tls_roots_password is missing"); - assertConfStrError("http::addr=localhost;tls_roots_password=hunter123;", "tls_roots_password was configured, but tls_roots is missing"); - assertConfStrError("tcp::addr=localhost;user=foo;", "token cannot be empty nor null"); - assertConfStrError("tcp::addr=localhost;username=foo;", "token cannot be empty nor null"); - assertConfStrError("tcp::addr=localhost;token=foo;", "TCP token is configured, but user is missing"); - assertConfStrError("http::addr=localhost;user=foo;", "password cannot be empty nor null"); - assertConfStrError("http::addr=localhost;username=foo;", "password cannot be empty nor null"); - assertConfStrError("http::addr=localhost;pass=foo;", "HTTP password is configured, but username is missing"); - assertConfStrError("http::addr=localhost;password=foo;", "HTTP password is configured, but username is missing"); - assertConfStrError("tcp::addr=localhost;pass=foo;", "password is not supported for TCP protocol"); - assertConfStrError("tcp::addr=localhost;password=foo;", "password is not supported for TCP protocol"); - assertConfStrError("tcp::addr=localhost;retry_timeout=;", "retry_timeout cannot be empty"); - assertConfStrError("tcp::addr=localhost;max_buf_size=;", "max_buf_size cannot be empty"); - assertConfStrError("tcp::addr=localhost;init_buf_size=;", "init_buf_size cannot be empty"); - assertConfStrError("http::addr=localhost:8080;tls_verify=unsafe_off;", "TLS validation disabled, but TLS was not enabled"); - assertConfStrError("http::addr=localhost:8080;tls_verify=bad;", "invalid tls_verify [value=bad, allowed-values=[on, unsafe_off]]"); - assertConfStrError("tcps::addr=localhost;pass=unsafe_off;", "password is not supported for TCP protocol"); - assertConfStrError("tcps::addr=localhost;password=unsafe_off;", "password is not supported for TCP protocol"); - assertConfStrError("http::addr=localhost:8080;max_buf_size=-32;", "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=-32, initialBufferCapacity=65536]"); - assertConfStrError("http::addr=localhost:8080;max_buf_size=notanumber;", "invalid max_buf_size [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;init_buf_size=notanumber;", "invalid init_buf_size [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;init_buf_size=-42;", "buffer capacity cannot be negative [capacity=-42]"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=0;", "invalid auto_flush_rows [value=0]"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=notanumber;", "invalid auto_flush_rows [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;auto_flush=invalid;", "invalid auto_flush [value=invalid, allowed-values=[on, off]]"); - assertConfStrError("http::addr=localhost:8080;auto_flush=off;auto_flush_rows=100;", "cannot set auto flush rows when auto-flush is already disabled"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=100;auto_flush=off;", "auto flush rows was already configured [autoFlushRows=100]"); - assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_interval=1;", "cannot set auto flush interval when interval based auto-flush is already disabled"); - assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_rows=1;", "cannot set auto flush rows when auto-flush is already disabled"); - assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport"); - assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset"); - assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]"); - - assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1"); - assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "auto_flush=on", "protocol_version=2"); - assertConfStrOk("addr=localhost", "auto_flush=on", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "max_name_len=1024", "protocol_version=2"); - - assertConfStrError("tcp::addr=localhost;auto_flush_bytes=1024;init_buf_size=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=2048, auto_flush_bytes=1024]"); - assertConfStrError("tcp::addr=localhost;init_buf_size=1024;auto_flush_bytes=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=1024, auto_flush_bytes=2048]"); - assertConfStrError("tcp::addr=localhost;auto_flush_bytes=off;", "TCP transport must have auto_flush_bytes enabled"); - - assertConfStrOk("http::addr=localhost;auto_flush=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_rows=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=1;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_rows=off;auto_flush_interval=1;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;auto_flush=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush=off;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); - assertConfStrOk("http::addr=localhost:8080;protocol_version=2;"); - assertConfStrOk("http::addr=localhost:8080;token=foo;protocol_version=2;"); - assertConfStrOk("http::addr=localhost:8080;token=foo=bar;protocol_version=2;"); - assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=1"); - assertConfStrOk("http::addr=localhost:8080;token=foo;max_buf_size=1000000;retry_timeout=1000;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;max_name_len=256;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=3;"); - assertConfStrError("https::addr=2001:0db8:85a3:0000:0000:8a2e:0370:7334;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=2001:0db8:85a3:0000:0000:8a2e:0370:7334]"); - assertConfStrError("https::addr=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000]"); - }); - } - - @Test - public void testCustomTruststoreButTlsNotEnabled() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) - .address(LOCALHOST); - try { - builder.build(); - fail("should fail when custom trust store configured, but TLS not enabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS was not enabled"); - } - }); - } - - @Test - public void testCustomTruststoreDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow double custom trust store set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testCustomTruststorePasswordCannotBeNull() { - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, null); - fail("should not allow null trust store password"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store password cannot be null"); - } - } - - @Test - public void testCustomTruststorePathCannotBeBlank() { - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore("", TRUSTSTORE_PASSWORD); - fail("should not allow blank trust store path"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store path cannot be empty nor null"); - } - - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(null, TRUSTSTORE_PASSWORD); - fail("should not allow null trust store path"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store path cannot be empty nor null"); - } - } - - @Test - public void testDisableAutoFlushNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.TCP).address(LOCALHOST).disableAutoFlush().build()) { - fail("TCP does not support disabling auto-flush"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto-flush is not supported for TCP protocol"); - } - }); - } - - @Test - public void testDnsResolutionFail() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.TCP).address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld").build()) { - fail("dns resolution errors should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not resolve"); - } - }); - } - - @Test - public void testDuplicatedAddresses() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9000"); - Assert.fail("should not allow multiple addresses"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "duplicated addresses are not allowed [address=localhost:9000]"); - } - - try { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001").address("localhost:9000"); - Assert.fail("should not allow multiple addresses"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "duplicated addresses are not allowed [address=localhost:9000]"); - } - }); - } - - @Test - public void testDuplicatedAddressesWithDifferentPortsAllowed() throws Exception { - assertMemoryLeak(() -> { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001"); - }); - } - - @Test - public void testDuplicatedAddressesWithNoPortsAllowed() throws Exception { - assertMemoryLeak(() -> { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost"); - Sender.builder(Sender.Transport.TCP).address("localhost").address("localhost:9000"); - }); - } - - @Test - public void testFailFastWhenSetCustomTrustStoreTwice() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow double custom trust store set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - } - - @Test - public void testFirstTlsValidationDisabledThenCustomTruststore() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation(); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow custom truststore when TLS validation was disabled disabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS validation was already disabled"); - } - }); - } - - @Test - public void testHostNorAddressSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.build(); - fail("not host should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "server address not set"); - } - }); - } - - @Test - public void testHttpTokenNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpToken("foo").build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP token authentication is not supported for TCP protocol"); - } - }); - } - - @Test - public void testInvalidHttpTimeout() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(0); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout must be positive [timeout=0]"); - } - - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(-1); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout must be positive [timeout=-1]"); - } - - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(100).httpTimeoutMillis(200); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout was already configured [timeout=100]"); - } - - try { - Sender.builder(Sender.Transport.TCP).address("localhost").httpTimeoutMillis(5000).build(); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout is not supported for TCP protocol"); - } - }); - } - - @Test - public void testInvalidRetryTimeout() { - try { - Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(-1); - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retry timeout cannot be negative [retryTimeoutMillis=-1]"); - } - - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(100); - try { - builder.retryTimeoutMillis(200); - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retry timeout was already configured [retryTimeoutMillis=100]"); - } - } - - @Test - public void testMalformedPortInAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address("foo:nonsense12334"); - fail("should fail with malformated port"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "cannot parse a port from the address"); - } - }); - } - - @Test - public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP) - .address("localhost:1") - .maxBufferCapacity(65535) - .build() - ) { - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]"); - } - }); - } - - @Test - public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP) - .address("localhost:1") - .maxBufferCapacity(100_000) - .bufferCapacity(200_000) - .build() - ) { - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=100000, initialBufferCapacity=200000]"); - } - }); - } - - @Test - public void testMaxRetriesNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100).build(); - fail("max retries should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retrying is not supported for TCP protocol"); - } - }); - } - - @Test - public void testMinRequestThroughputCannotBeNegative() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100).build(); - fail("minimum request throughput must not be negative"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimum request throughput must not be negative [minRequestThroughput=-100]"); - } - }); - } - - @Test - public void testMinRequestThroughputNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).minRequestThroughput(1).build(); - fail("min request throughput is not be supported for TCP and the builder should fail-fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimum request throughput is not supported for TCP protocol"); - } - }); - } - - @Test - public void testPlainAuth_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testPlainOldTokenNotSupportedForHttpProtocol() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address("localhost:9000").enableAuth("key").authToken(AUTH_TOKEN_KEY1).build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "old token authentication is not supported for HTTP protocol"); - } - }); - } - - @Test - public void testPlain_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testPortDoubleSet_firstAddressThenPort() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":9000"); - try { - builder.port(9000); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testPortDoubleSet_firstPortThenAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).port(9000); - try { - builder.address(LOCALHOST + ":9000"); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testPortDoubleSet_firstPortThenPort() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).port(9000); - try { - builder.port(9000); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "questdb server address not set"); - } - }); - } - - @Test - public void testSmallMaxNameLen() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.LineSenderBuilder ignored = Sender - .builder(Sender.Transport.TCP) - .maxNameLength(10); - fail("should not allow double buffer capacity set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "max_name_len must be at least 16 bytes [max_name_len=10]"); - } - }); - } - - @Test - public void testTlsDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableTls(); - try { - builder.enableTls(); - fail("should not allow double tls set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already enabled"); - } - }); - } - - @Test - public void testTlsValidationDisabledButTlsNotEnabled() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation() - .address(LOCALHOST); - try { - builder.build(); - fail("should fail when TLS validation is disabled, but TLS not enabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS was not enabled"); - } - }); - } - - @Test - public void testTlsValidationDisabledDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation(); - try { - builder.advancedTls().disableCertificateValidation(); - fail("should not allow double TLS validation disabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS validation was already disabled"); - } - }); - } - - @Test - public void testTls_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableTls().address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testUsernamePasswordAuthNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpUsernamePassword("foo", "bar").build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "username/password authentication is not supported for TCP protocol"); - } - }); - } - - private static void assertConfStrError(String conf, String expectedError) { - try { - try (Sender ignored = Sender.fromConfig(conf)) { - fail("should fail with bad conf string"); - } - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), expectedError); - } - } - - private static void assertConfStrOk(String... params) { - StringBuilder sb = new StringBuilder(); - sb.append("http").append("::"); - shuffle(params); - for (int i = 0; i < params.length; i++) { - sb.append(params[i]).append(";"); - } - assertConfStrOk(sb.toString()); - } - - private static void assertConfStrOk(String conf) { - Sender.fromConfig(conf).close(); - } - - private static void shuffle(String[] input) { - for (int i = 0; i < input.length; i++) { - int j = (int) (Math.random() * input.length); - String tmp = input[i]; - input[i] = input[j]; - input[j] = tmp; - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java new file mode 100644 index 0000000..bf9f5c7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -0,0 +1,490 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.line; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Test; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import static org.junit.Assert.fail; + +/** + * Unit tests for LineSenderBuilder that don't require a running QuestDB instance. + * Tests that require an actual QuestDB connection have been moved to integration tests. + */ +public class LineSenderBuilderTest { + private static final String AUTH_TOKEN_KEY1 = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; + private static final String LOCALHOST = "localhost"; + private static final char[] TRUSTSTORE_PASSWORD = "questdb".toCharArray(); + private static final String TRUSTSTORE_PATH = "/keystore/server.keystore"; + + @Test + public void testAddressDoubleSet_firstAddressThenAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).address("127.0.0.1"))); + } + + @Test + public void testAddressEmpty() throws Exception { + assertMemoryLeak(() -> assertThrows("address cannot be empty", + () -> Sender.builder(Sender.Transport.TCP).address(""))); + } + + @Test + public void testAddressEndsWithColon() throws Exception { + assertMemoryLeak(() -> assertThrows("invalid address", + () -> Sender.builder(Sender.Transport.TCP).address("foo:"))); + } + + @Test + public void testAddressNull() throws Exception { + assertMemoryLeak(() -> assertThrows("null", + () -> Sender.builder(Sender.Transport.TCP).address(null))); + } + + @Test + public void testAuthDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken(AUTH_TOKEN_KEY1).enableAuth("bar"))); + } + + @Test + public void testAuthTooSmallBuffer() throws Exception { + assertMemoryLeak(() -> assertThrows("minimalCapacity", + Sender.builder(Sender.Transport.TCP) + .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":9001") + .bufferCapacity(1))); + } + + @Test + public void testAuthWithBadToken() throws Exception { + assertMemoryLeak(() -> assertThrows("could not import token", + () -> Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken("bar token"))); + } + + @Test + public void testAutoFlushIntervalMustBePositive() { + assertThrows("auto flush interval cannot be negative [autoFlushIntervalMillis=0]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(0)); + assertThrows("auto flush interval cannot be negative [autoFlushIntervalMillis=-1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(-1)); + } + + @Test + public void testAutoFlushIntervalNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush interval is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushInterval_afterAutoFlushDisabled() throws Exception { + assertMemoryLeak(() -> assertThrows("cannot set auto flush interval when interval based auto-flush is already disabled", + () -> Sender.builder(Sender.Transport.HTTP).disableAutoFlush().autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushInterval_doubleConfiguration() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush interval was already configured [autoFlushIntervalMillis=1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(1).autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushRowsCannotBeNegative() { + assertThrows("auto flush rows cannot be negative [autoFlushRows=-1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushRows(-1)); + } + + @Test + public void testAutoFlushRowsNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush rows is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushRows(1))); + } + + @Test + public void testAutoFlushRows_doubleConfiguration() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush rows was already configured [autoFlushRows=1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushRows(1).autoFlushRows(1))); + } + + @Test + public void testBufferSizeDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP).bufferCapacity(1024).bufferCapacity(1024))); + } + + @Test + public void testConfStringValidation() throws Exception { + assertMemoryLeak(() -> { + assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("http::addr=localhost:-1;", "invalid port [port=-1]"); + assertConfStrError("http::auto_flush=on;", "addr is missing"); + assertConfStrError("http::addr=localhost;tls_roots=/some/path;", "tls_roots was configured, but tls_roots_password is missing"); + assertConfStrError("http::addr=localhost;tls_roots_password=hunter123;", "tls_roots_password was configured, but tls_roots is missing"); + assertConfStrError("tcp::addr=localhost;user=foo;", "token cannot be empty nor null"); + assertConfStrError("tcp::addr=localhost;username=foo;", "token cannot be empty nor null"); + assertConfStrError("tcp::addr=localhost;token=foo;", "TCP token is configured, but user is missing"); + assertConfStrError("http::addr=localhost;user=foo;", "password cannot be empty nor null"); + assertConfStrError("http::addr=localhost;username=foo;", "password cannot be empty nor null"); + assertConfStrError("http::addr=localhost;pass=foo;", "HTTP password is configured, but username is missing"); + assertConfStrError("http::addr=localhost;password=foo;", "HTTP password is configured, but username is missing"); + assertConfStrError("tcp::addr=localhost;pass=foo;", "password is not supported for TCP protocol"); + assertConfStrError("tcp::addr=localhost;password=foo;", "password is not supported for TCP protocol"); + assertConfStrError("tcp::addr=localhost;retry_timeout=;", "retry_timeout cannot be empty"); + assertConfStrError("tcp::addr=localhost;max_buf_size=;", "max_buf_size cannot be empty"); + assertConfStrError("tcp::addr=localhost;init_buf_size=;", "init_buf_size cannot be empty"); + assertConfStrError("http::addr=localhost:8080;tls_verify=unsafe_off;", "TLS validation disabled, but TLS was not enabled"); + assertConfStrError("http::addr=localhost:8080;tls_verify=bad;", "invalid tls_verify [value=bad, allowed-values=[on, unsafe_off]]"); + assertConfStrError("tcps::addr=localhost;pass=unsafe_off;", "password is not supported for TCP protocol"); + assertConfStrError("tcps::addr=localhost;password=unsafe_off;", "password is not supported for TCP protocol"); + assertConfStrError("http::addr=localhost:8080;max_buf_size=-32;", "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=-32, initialBufferCapacity=65536]"); + assertConfStrError("http::addr=localhost:8080;max_buf_size=notanumber;", "invalid max_buf_size [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;init_buf_size=notanumber;", "invalid init_buf_size [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;init_buf_size=-42;", "buffer capacity cannot be negative [capacity=-42]"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=0;", "invalid auto_flush_rows [value=0]"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=notanumber;", "invalid auto_flush_rows [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;auto_flush=invalid;", "invalid auto_flush [value=invalid, allowed-values=[on, off]]"); + assertConfStrError("http::addr=localhost:8080;auto_flush=off;auto_flush_rows=100;", "cannot set auto flush rows when auto-flush is already disabled"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=100;auto_flush=off;", "auto flush rows was already configured [autoFlushRows=100]"); + assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_interval=1;", "cannot set auto flush interval when interval based auto-flush is already disabled"); + assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_rows=1;", "cannot set auto flush rows when auto-flush is already disabled"); + assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport"); + assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset"); + assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]"); + + assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1"); + assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "auto_flush=on", "protocol_version=2"); + assertConfStrOk("addr=localhost", "auto_flush=on", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "max_name_len=1024", "protocol_version=2"); + + assertConfStrError("tcp::addr=localhost;auto_flush_bytes=1024;init_buf_size=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=2048, auto_flush_bytes=1024]"); + assertConfStrError("tcp::addr=localhost;init_buf_size=1024;auto_flush_bytes=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=1024, auto_flush_bytes=2048]"); + assertConfStrError("tcp::addr=localhost;auto_flush_bytes=off;", "TCP transport must have auto_flush_bytes enabled"); + + assertConfStrOk("http::addr=localhost;auto_flush=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_rows=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=1;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_rows=off;auto_flush_interval=1;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;auto_flush=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush=off;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); + assertConfStrOk("http::addr=localhost:8080;protocol_version=2;"); + assertConfStrOk("http::addr=localhost:8080;token=foo;protocol_version=2;"); + assertConfStrOk("http::addr=localhost:8080;token=foo=bar;protocol_version=2;"); + assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=1"); + assertConfStrOk("http::addr=localhost:8080;token=foo;max_buf_size=1000000;retry_timeout=1000;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;max_name_len=256;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=3;"); + assertConfStrError("https::addr=2001:0db8:85a3:0000:0000:8a2e:0370:7334;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=2001:0db8:85a3:0000:0000:8a2e:0370:7334]"); + assertConfStrError("https::addr=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000]"); + }); + } + + @Test + public void testCustomTruststoreButTlsNotEnabled() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS was not enabled", + Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .address(LOCALHOST))); + } + + @Test + public void testCustomTruststoreDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD))); + } + + @Test + public void testCustomTruststorePasswordCannotBeNull() { + assertThrows("trust store password cannot be null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, null)); + } + + @Test + public void testCustomTruststorePathCannotBeBlank() { + assertThrows("trust store path cannot be empty nor null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore("", TRUSTSTORE_PASSWORD)); + assertThrows("trust store path cannot be empty nor null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(null, TRUSTSTORE_PASSWORD)); + } + + @Test + public void testDisableAutoFlushNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto-flush is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).disableAutoFlush())); + } + + @Test + public void testDnsResolutionFail() throws Exception { + assertMemoryLeak(() -> assertThrows("could not resolve", + Sender.builder(Sender.Transport.TCP).address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld"))); + } + + @Test + public void testDuplicatedAddresses() throws Exception { + assertMemoryLeak(() -> { + assertThrows("duplicated addresses are not allowed [address=localhost:9000]", + () -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9000")); + assertThrows("duplicated addresses are not allowed [address=localhost:9000]", + () -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001").address("localhost:9000")); + }); + } + + @Test + public void testDuplicatedAddressesWithDifferentPortsAllowed() throws Exception { + assertMemoryLeak(() -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001")); + } + + @Test + public void testDuplicatedAddressesWithNoPortsAllowed() throws Exception { + assertMemoryLeak(() -> { + Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost"); + Sender.builder(Sender.Transport.TCP).address("localhost").address("localhost:9000"); + }); + } + + @Test + public void testFailFastWhenSetCustomTrustStoreTwice() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD)); + } + + @Test + public void testFirstTlsValidationDisabledThenCustomTruststore() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS validation was already disabled", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD))); + } + + @Test + public void testHostNorAddressSet() throws Exception { + assertMemoryLeak(() -> assertThrows("server address not set", + Sender.builder(Sender.Transport.TCP))); + } + + @Test + public void testHttpTokenNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("HTTP token authentication is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpToken("foo"))); + } + + @Test + public void testInvalidHttpTimeout() throws Exception { + assertMemoryLeak(() -> { + assertThrows("HTTP timeout must be positive [timeout=0]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(0)); + assertThrows("HTTP timeout must be positive [timeout=-1]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(-1)); + assertThrows("HTTP timeout was already configured [timeout=100]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(100).httpTimeoutMillis(200)); + assertThrows("HTTP timeout is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address("localhost").httpTimeoutMillis(5000)); + }); + } + + @Test + public void testInvalidRetryTimeout() { + assertThrows("retry timeout cannot be negative [retryTimeoutMillis=-1]", + () -> Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(-1)); + assertThrows("retry timeout was already configured [retryTimeoutMillis=100]", + () -> Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(100).retryTimeoutMillis(200)); + } + + @Test + public void testMalformedPortInAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("cannot parse a port from the address", + () -> Sender.builder(Sender.Transport.TCP).address("foo:nonsense12334"))); + } + + @Test + public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { + assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]", + Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(65535))); + } + + @Test + public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws Exception { + assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=100000, initialBufferCapacity=200000]", + Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(100_000).bufferCapacity(200_000))); + } + + @Test + public void testMaxRetriesNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("retrying is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100))); + } + + @Test + public void testMinRequestThroughputCannotBeNegative() throws Exception { + assertMemoryLeak(() -> assertThrows("minimum request throughput must not be negative [minRequestThroughput=-100]", + Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100))); + } + + @Test + public void testMinRequestThroughputNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("minimum request throughput is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).minRequestThroughput(1))); + } + + @Test + public void testPlainAuth_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP) + .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":19003"))); + } + + @Test + public void testPlainOldTokenNotSupportedForHttpProtocol() throws Exception { + assertMemoryLeak(() -> assertThrows("old token authentication is not supported for HTTP protocol", + Sender.builder(Sender.Transport.HTTP).address("localhost:9000").enableAuth("key").authToken(AUTH_TOKEN_KEY1))); + } + + @Test + public void testPlain_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":19003"))); + } + + @Test + public void testPortDoubleSet_firstAddressThenPort() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":9000").port(9000))); + } + + @Test + public void testPortDoubleSet_firstPortThenAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).port(9000).address(LOCALHOST + ":9000"))); + } + + @Test + public void testPortDoubleSet_firstPortThenPort() throws Exception { + assertMemoryLeak(() -> assertThrows("questdb server address not set", + Sender.builder(Sender.Transport.TCP).port(9000).port(9000))); + } + + @Test + public void testSmallMaxNameLen() throws Exception { + assertMemoryLeak(() -> assertThrows("max_name_len must be at least 16 bytes [max_name_len=10]", + () -> Sender.builder(Sender.Transport.TCP).maxNameLength(10))); + } + + @Test + public void testTlsDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already enabled", + () -> Sender.builder(Sender.Transport.TCP).enableTls().enableTls())); + } + + @Test + public void testTlsValidationDisabledButTlsNotEnabled() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS was not enabled", + Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .address(LOCALHOST))); + } + + @Test + public void testTlsValidationDisabledDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS validation was already disabled", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .advancedTls().disableCertificateValidation())); + } + + @Test + public void testTls_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP).enableTls().address(LOCALHOST + ":19003"))); + } + + @Test + public void testUsernamePasswordAuthNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("username/password authentication is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpUsernamePassword("foo", "bar"))); + } + + private static void assertConfStrError(String conf, String expectedError) { + try { + try (Sender ignored = Sender.fromConfig(conf)) { + fail("should fail with bad conf string"); + } + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedError); + } + } + + private static void assertConfStrOk(String... params) { + StringBuilder sb = new StringBuilder(); + sb.append("http").append("::"); + shuffle(params); + for (int i = 0; i < params.length; i++) { + sb.append(params[i]).append(";"); + } + assertConfStrOk(sb.toString()); + } + + private static void assertConfStrOk(String conf) { + Sender.fromConfig(conf).close(); + } + + private static void assertThrows(String expectedSubstring, Sender.LineSenderBuilder builder) { + assertThrows(expectedSubstring, builder::build); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void shuffle(String[] input) { + for (int i = 0; i < input.length; i++) { + int j = (int) (Math.random() * input.length); + String tmp = input[i]; + input[i] = input[j]; + input[j] = tmp; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java similarity index 99% rename from core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java rename to core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java index 736aec4..92c79d0 100644 --- a/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java @@ -22,10 +22,11 @@ * ******************************************************************************/ -package io.questdb.client.test; +package io.questdb.client.test.cutlass.line; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.AbstractTest; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Ignore; diff --git a/core/src/test/java/module-info.java b/core/src/test/java/module-info.java index 3e33568..86341d8 100644 --- a/core/src/test/java/module-info.java +++ b/core/src/test/java/module-info.java @@ -35,4 +35,5 @@ exports io.questdb.client.test; exports io.questdb.client.test.cairo; + exports io.questdb.client.test.cutlass.line; } From 49ecfa727e2b0d1b9f2a7102cf37e06086857cb1 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 22:35:15 +0000 Subject: [PATCH 005/230] ilpv4 -> QWP (QuestDB Wire Protocol) --- .../main/java/io/questdb/client/Sender.java | 6 +- .../cutlass/http/client/WebSocketClient.java | 8 +- .../http/client/WebSocketSendBuffer.java | 12 +- .../client/GlobalSymbolDictionary.java | 2 +- .../{ilpv4 => qwp}/client/InFlightWindow.java | 2 +- .../client/MicrobatchBuffer.java | 4 +- .../client/NativeBufferWriter.java | 4 +- .../client/QwpBufferWriter.java} | 4 +- .../client/QwpWebSocketEncoder.java} | 72 +- .../client/QwpWebSocketSender.java} | 154 ++-- .../{ilpv4 => qwp}/client/ResponseReader.java | 2 +- .../client/WebSocketChannel.java | 12 +- .../client/WebSocketResponse.java | 2 +- .../client/WebSocketSendQueue.java | 2 +- .../protocol/QwpBitReader.java} | 8 +- .../protocol/QwpBitWriter.java} | 8 +- .../protocol/QwpColumnDef.java} | 18 +- .../protocol/QwpConstants.java} | 6 +- .../protocol/QwpGorillaEncoder.java} | 10 +- .../protocol/QwpNullBitmap.java} | 6 +- .../protocol/QwpSchemaHash.java} | 10 +- .../protocol/QwpTableBuffer.java} | 20 +- .../protocol/QwpVarint.java} | 6 +- .../protocol/QwpZigZag.java} | 6 +- .../websocket/WebSocketCloseCode.java | 2 +- .../websocket/WebSocketFrameParser.java | 2 +- .../websocket/WebSocketFrameWriter.java | 2 +- .../websocket/WebSocketHandshake.java | 2 +- .../websocket/WebSocketOpcode.java | 2 +- core/src/main/java/module-info.java | 6 +- .../qwp/client/InFlightWindowTest.java | 832 ++++++++++++++++++ 31 files changed, 1032 insertions(+), 200 deletions(-) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/GlobalSymbolDictionary.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/InFlightWindow.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/MicrobatchBuffer.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/NativeBufferWriter.java (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/client/IlpBufferWriter.java => qwp/client/QwpBufferWriter.java} (97%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/client/IlpV4WebSocketEncoder.java => qwp/client/QwpWebSocketEncoder.java} (90%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/client/IlpV4WebSocketSender.java => qwp/client/QwpWebSocketSender.java} (87%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/ResponseReader.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/WebSocketChannel.java (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/WebSocketResponse.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/WebSocketSendQueue.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4BitReader.java => qwp/protocol/QwpBitReader.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4BitWriter.java => qwp/protocol/QwpBitWriter.java} (97%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4ColumnDef.java => qwp/protocol/QwpColumnDef.java} (89%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4Constants.java => qwp/protocol/QwpConstants.java} (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4GorillaEncoder.java => qwp/protocol/QwpGorillaEncoder.java} (97%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4NullBitmap.java => qwp/protocol/QwpNullBitmap.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4SchemaHash.java => qwp/protocol/QwpSchemaHash.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4TableBuffer.java => qwp/protocol/QwpTableBuffer.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4Varint.java => qwp/protocol/QwpVarint.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4ZigZag.java => qwp/protocol/QwpZigZag.java} (96%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketCloseCode.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketFrameParser.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketFrameWriter.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketHandshake.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketOpcode.java (98%) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index fcc6d46..b2b3c19 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -34,7 +34,7 @@ import io.questdb.client.cutlass.line.http.AbstractLineHttpSender; import io.questdb.client.cutlass.line.tcp.DelegatingTlsChannel; import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; -import io.questdb.client.cutlass.ilpv4.client.IlpV4WebSocketSender; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; @@ -881,7 +881,7 @@ public Sender build() { int actualSendQueueCapacity = sendQueueCapacity == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEND_QUEUE_CAPACITY : sendQueueCapacity; if (asyncMode) { - return IlpV4WebSocketSender.connectAsync( + return QwpWebSocketSender.connectAsync( hosts.getQuick(0), ports.getQuick(0), tlsEnabled, @@ -892,7 +892,7 @@ public Sender build() { actualSendQueueCapacity ); } else { - return IlpV4WebSocketSender.connect( + return QwpWebSocketSender.connect( hosts.getQuick(0), ports.getQuick(0), tlsEnabled diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index cc64a63..ab8f696 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -25,10 +25,10 @@ package io.questdb.client.cutlass.http.client; import io.questdb.client.HttpClientConfiguration; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.network.IOOperation; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.Socket; diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 20c43c1..5075b9f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -24,9 +24,9 @@ package io.questdb.client.cutlass.http.client; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; -import io.questdb.client.cutlass.ilpv4.client.IlpBufferWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Numbers; @@ -55,7 +55,7 @@ *

* Thread safety: This class is NOT thread-safe. Each connection should have its own buffer. */ -public class WebSocketSendBuffer implements IlpBufferWriter, QuietCloseable { +public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable { // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key) private static final int MAX_HEADER_SIZE = 14; @@ -244,7 +244,7 @@ public void putAscii(CharSequence cs) { writePos += len; } - // === IlpBufferWriter Implementation === + // === QwpBufferWriter Implementation === /** * Writes an unsigned variable-length integer (LEB128 encoding). @@ -267,7 +267,7 @@ public void putString(String value) { putVarint(0); return; } - int utf8Len = IlpBufferWriter.utf8Length(value); + int utf8Len = QwpBufferWriter.utf8Length(value); putVarint(utf8Len); putUtf8(value); } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 743c029..4d75211 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.ObjList; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java index 9db5456..cffd93e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.line.LineSenderException; import org.slf4j.Logger; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 832bb66..9bcf738 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; @@ -35,7 +35,7 @@ * A buffer for accumulating ILP data into microbatches before sending. *

* This class implements a state machine for buffer lifecycle management in the - * double-buffering scheme used by {@link IlpV4WebSocketSender}: + * double-buffering scheme used by {@link QwpWebSocketSender}: * 

  * Buffer States:
  * ┌─────────────┐    seal()     ┌─────────────┐    markSending()  ┌─────────────┐
diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
similarity index 98%
rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java
rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
index 5cab035..ee264ed 100644
--- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
@@ -22,7 +22,7 @@
  *
  ******************************************************************************/
 
-package io.questdb.client.cutlass.ilpv4.client;
+package io.questdb.client.cutlass.qwp.client;
 
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
@@ -36,7 +36,7 @@
  * 

  * All multi-byte values are written in little-endian format unless otherwise specified.
  */
-public class NativeBufferWriter implements IlpBufferWriter, QuietCloseable {
+public class NativeBufferWriter implements QwpBufferWriter, QuietCloseable {
 
     private static final int DEFAULT_CAPACITY = 8192;
 
diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
similarity index 97%
rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java
rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
index 1976cac..05ef0ea 100644
--- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
@@ -22,7 +22,7 @@
  *
  ******************************************************************************/
 
-package io.questdb.client.cutlass.ilpv4.client;
+package io.questdb.client.cutlass.qwp.client;
 
 import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
 
@@ -42,7 +42,7 @@
  * All multi-byte values are written in little-endian format unless the method
  * name explicitly indicates big-endian (e.g., {@link #putLongBE}).
  */
-public interface IlpBufferWriter extends ArrayBufferAppender {
+public interface QwpBufferWriter extends ArrayBufferAppender {
 
     // === Primitive writes (little-endian) ===
 
diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
similarity index 90%
rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java
rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
index e98a1df..c71e158 100644
--- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
@@ -22,20 +22,20 @@
  *
  ******************************************************************************/
 
-package io.questdb.client.cutlass.ilpv4.client;
+package io.questdb.client.cutlass.qwp.client;
 
-import io.questdb.client.cutlass.ilpv4.protocol.IlpV4ColumnDef;
-import io.questdb.client.cutlass.ilpv4.protocol.IlpV4GorillaEncoder;
-import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TableBuffer;
+import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
+import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder;
+import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
 import io.questdb.client.std.QuietCloseable;
 
-import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*;
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
 
 /**
  * Encodes ILP v4 messages for WebSocket transport.
  * 

  * This encoder can write to either an internal {@link NativeBufferWriter} (default)
- * or an external {@link IlpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}.
+ * or an external {@link QwpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}.
  * 

  * When using an external buffer, the encoder writes directly to it without intermediate copies,
  * enabling zero-copy WebSocket frame construction.
@@ -50,7 +50,7 @@
  * client.sendFrame(frame);
  *
*/ -public class IlpV4WebSocketEncoder implements QuietCloseable { +public class QwpWebSocketEncoder implements QuietCloseable { /** * Encoding flag for Gorilla-encoded timestamps. @@ -60,18 +60,18 @@ public class IlpV4WebSocketEncoder implements QuietCloseable { * Encoding flag for uncompressed timestamps. */ public static final byte ENCODING_UNCOMPRESSED = 0x00; - private final IlpV4GorillaEncoder gorillaEncoder = new IlpV4GorillaEncoder(); - private IlpBufferWriter buffer; + private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); + private QwpBufferWriter buffer; private byte flags; private NativeBufferWriter ownedBuffer; - public IlpV4WebSocketEncoder() { + public QwpWebSocketEncoder() { this.ownedBuffer = new NativeBufferWriter(); this.buffer = ownedBuffer; this.flags = 0; } - public IlpV4WebSocketEncoder(int bufferSize) { + public QwpWebSocketEncoder(int bufferSize) { this.ownedBuffer = new NativeBufferWriter(bufferSize); this.buffer = ownedBuffer; this.flags = 0; @@ -92,7 +92,7 @@ public void close() { * @param useSchemaRef whether to use schema reference mode * @return the number of bytes written */ - public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { buffer.reset(); // Write message header with placeholder for payload length @@ -123,7 +123,7 @@ public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { * @return the number of bytes written */ public int encodeWithDeltaDict( - IlpV4TableBuffer tableBuffer, + QwpTableBuffer tableBuffer, GlobalSymbolDictionary globalDict, int confirmedMaxId, int batchMaxId, @@ -167,10 +167,10 @@ public int encodeWithDeltaDict( /** * Returns the underlying buffer. *

- * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, + * If an external buffer was set via {@link #setBuffer(QwpBufferWriter)}, * that buffer is returned. Otherwise, returns the internal buffer. */ - public IlpBufferWriter getBuffer() { + public QwpBufferWriter getBuffer() { return buffer; } @@ -218,7 +218,7 @@ public void reset() { * * @param externalBuffer the external buffer to use, or null to use internal buffer */ - public void setBuffer(IlpBufferWriter externalBuffer) { + public void setBuffer(QwpBufferWriter externalBuffer) { this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; } @@ -273,7 +273,7 @@ public void writeHeader(int tableCount, int payloadLength) { /** * Encodes a single column. */ - private void encodeColumn(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); // Write null bitmap if column is nullable @@ -351,7 +351,7 @@ private void encodeColumn(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colD * Encodes a single column using global symbol IDs for SYMBOL type. * All other column types are encoded the same as encodeColumn. */ - private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); // Write null bitmap if column is nullable @@ -430,8 +430,8 @@ private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, Il /** * Encodes a single table from the buffer. */ - private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); if (useSchemaRef) { @@ -448,8 +448,8 @@ private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + QwpColumnDef colDef = columnDefs[i]; encodeColumn(col, colDef, rowCount, useGorilla); } } @@ -458,8 +458,8 @@ private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { * Encodes a single table from the buffer using global symbol IDs. * This is used with delta dictionary encoding. */ - private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); if (useSchemaRef) { @@ -476,8 +476,8 @@ private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + QwpColumnDef colDef = columnDefs[i]; encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); } } @@ -531,7 +531,7 @@ private void writeDecimal64Column(byte scale, long[] values, int count) { } } - private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); double[] data = col.getDoubleArrayData(); @@ -581,7 +581,7 @@ private void writeLong256Column(long[] values, int count) { } } - private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); long[] data = col.getLongArrayData(); @@ -639,7 +639,7 @@ private void writeStringColumn(String[] strings, int count) { int totalDataLen = 0; for (int i = 0; i < count; i++) { if (strings[i] != null) { - totalDataLen += IlpBufferWriter.utf8Length(strings[i]); + totalDataLen += QwpBufferWriter.utf8Length(strings[i]); } } @@ -648,7 +648,7 @@ private void writeStringColumn(String[] strings, int count) { buffer.putInt(0); for (int i = 0; i < count; i++) { if (strings[i] != null) { - runningOffset += IlpBufferWriter.utf8Length(strings[i]); + runningOffset += QwpBufferWriter.utf8Length(strings[i]); } buffer.putInt(runningOffset); } @@ -668,7 +668,7 @@ private void writeStringColumn(String[] strings, int count) { * - Dictionary entries (length-prefixed UTF-8 strings) * - Symbol indices (varints, one per value) */ - private void writeSymbolColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) { // Get symbol data from column buffer int[] symbolIndices = col.getSymbolIndices(); String[] dictionary = col.getSymbolDictionary(); @@ -693,7 +693,7 @@ private void writeSymbolColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { * The dictionary is not included here because it's written at the message level * in delta format. */ - private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) { int[] globalIds = col.getGlobalSymbolIds(); if (globalIds == null) { // Fall back to local indices if no global IDs stored @@ -712,7 +712,7 @@ private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, i /** * Writes a table header with full schema. */ - private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { + private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) { // Table name buffer.putString(tableName); @@ -726,7 +726,7 @@ private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4Col buffer.putByte(SCHEMA_MODE_FULL); // Column definitions (name + type for each) - for (IlpV4ColumnDef col : columns) { + for (QwpColumnDef col : columns) { buffer.putString(col.getName()); buffer.putByte(col.getWireTypeCode()); } @@ -760,12 +760,12 @@ private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long * Otherwise, falls back to uncompressed encoding. */ private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { - if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { + if (useGorilla && count > 2 && QwpGorillaEncoder.canUseGorilla(values, count)) { // Write Gorilla encoding flag buffer.putByte(ENCODING_GORILLA); // Calculate size needed and ensure buffer has capacity - int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); buffer.ensureCapacity(encodedSize); // Encode timestamps to buffer diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java similarity index 87% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 43a0b4c..6b0343f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -22,9 +22,9 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; -import io.questdb.client.cutlass.ilpv4.protocol.*; +import io.questdb.client.cutlass.qwp.protocol.*; import io.questdb.client.Sender; import io.questdb.client.cutlass.http.client.WebSocketClient; @@ -50,7 +50,7 @@ import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; -import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * ILP v4 WebSocket client sender for streaming data to QuestDB. @@ -72,7 +72,7 @@ *

* Example usage: * 

- * try (IlpV4WebSocketSender sender = IlpV4WebSocketSender.connect("localhost", 9000)) {
+ * try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", 9000)) {
  *     for (int i = 0; i < 100_000; i++) {
  *         sender.table("metrics")
  *               .symbol("host", "server-" + (i % 10))
@@ -85,9 +85,9 @@
  * }
  *
*/ -public class IlpV4WebSocketSender implements Sender { +public class QwpWebSocketSender implements Sender { - private static final Logger LOG = LoggerFactory.getLogger(IlpV4WebSocketSender.class); + private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB @@ -101,15 +101,15 @@ public class IlpV4WebSocketSender implements Sender { private final String host; private final int port; private final boolean tlsEnabled; - private final CharSequenceObjHashMap tableBuffers; - private IlpV4TableBuffer currentTableBuffer; + private final CharSequenceObjHashMap tableBuffers; + private QwpTableBuffer currentTableBuffer; private String currentTableName; // Cached column references to avoid repeated hashmap lookups - private IlpV4TableBuffer.ColumnBuffer cachedTimestampColumn; - private IlpV4TableBuffer.ColumnBuffer cachedTimestampNanosColumn; + private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; + private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; // Encoder for ILP v4 messages - private final IlpV4WebSocketEncoder encoder; + private final QwpWebSocketEncoder encoder; // WebSocket client (zero-GC native implementation) private WebSocketClient client; @@ -159,13 +159,13 @@ public class IlpV4WebSocketSender implements Sender { // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. private final LongHashSet sentSchemaHashes = new LongHashSet(); - private IlpV4WebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, + private QwpWebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, int inFlightWindowSize, int sendQueueCapacity) { this.host = host; this.port = port; this.tlsEnabled = tlsEnabled; - this.encoder = new IlpV4WebSocketEncoder(bufferSize); + this.encoder = new QwpWebSocketEncoder(bufferSize); this.tableBuffers = new CharSequenceObjHashMap<>(); this.currentTableBuffer = null; this.currentTableName = null; @@ -197,7 +197,7 @@ private IlpV4WebSocketSender(String host, int port, boolean tlsEnabled, int buff * @param port server HTTP port (WebSocket upgrade happens on same port) * @return connected sender */ - public static IlpV4WebSocketSender connect(String host, int port) { + public static QwpWebSocketSender connect(String host, int port) { return connect(host, port, false); } @@ -210,8 +210,8 @@ public static IlpV4WebSocketSender connect(String host, int port) { * @param tlsEnabled whether to use TLS * @return connected sender */ - public static IlpV4WebSocketSender connect(String host, int port, boolean tlsEnabled) { - IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled) { + QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, 0, 0, 0, // No auto-flush in sync mode 1, 1 // window=1 for sync behavior, queue=1 (not used) @@ -231,7 +231,7 @@ public static IlpV4WebSocketSender connect(String host, int port, boolean tlsEna * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @return connected sender */ - public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos) { return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, @@ -251,11 +251,11 @@ public static IlpV4WebSocketSender connectAsync(String host, int port, boolean t * @param sendQueueCapacity max batches waiting to send (default: 16) * @return connected sender */ - public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, int inFlightWindowSize, int sendQueueCapacity) { - IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, sendQueueCapacity @@ -272,7 +272,7 @@ public static IlpV4WebSocketSender connectAsync(String host, int port, boolean t * @param tlsEnabled whether to use TLS * @return connected sender */ - public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { + public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { return connectAsync(host, port, tlsEnabled, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS); } @@ -280,7 +280,7 @@ public static IlpV4WebSocketSender connectAsync(String host, int port, boolean t /** * Factory method for SenderBuilder integration. */ - public static IlpV4WebSocketSender create( + public static QwpWebSocketSender create( String host, int port, boolean tlsEnabled, @@ -289,7 +289,7 @@ public static IlpV4WebSocketSender create( String username, String password ) { - IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, bufferSize, 0, 0, 0, 1, 1 // window=1 for sync behavior @@ -309,8 +309,8 @@ public static IlpV4WebSocketSender create( * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async * @return unconnected sender */ - public static IlpV4WebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { - return new IlpV4WebSocketSender( + public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { + return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, 0, 0, 0, inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY @@ -330,11 +330,11 @@ public static IlpV4WebSocketSender createForTesting(String host, int port, int i * @param sendQueueCapacity max batches waiting to send * @return unconnected sender */ - public static IlpV4WebSocketSender createForTesting( + public static QwpWebSocketSender createForTesting( String host, int port, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, int inFlightWindowSize, int sendQueueCapacity) { - return new IlpV4WebSocketSender( + return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, sendQueueCapacity @@ -395,7 +395,7 @@ public boolean isGorillaEnabled() { /** * Sets whether to use Gorilla timestamp encoding. */ - public IlpV4WebSocketSender setGorillaEnabled(boolean enabled) { + public QwpWebSocketSender setGorillaEnabled(boolean enabled) { this.gorillaEnabled = enabled; this.encoder.setGorillaEnabled(enabled); return this; @@ -469,9 +469,9 @@ public int getMaxSentSymbolId() { // // Usage: // // Setup (once) - // IlpV4TableBuffer tableBuffer = sender.getTableBuffer("q"); - // IlpV4TableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); - // IlpV4TableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); + // QwpTableBuffer tableBuffer = sender.getTableBuffer("q"); + // QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); + // QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); // // // Hot path (per row) // colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol)); @@ -483,10 +483,10 @@ public int getMaxSentSymbolId() { * Gets or creates a table buffer for direct access. * For high-throughput generators that want to bypass fluent API overhead. */ - public IlpV4TableBuffer getTableBuffer(String tableName) { - IlpV4TableBuffer buffer = tableBuffers.get(tableName); + public QwpTableBuffer getTableBuffer(String tableName) { + QwpTableBuffer buffer = tableBuffers.get(tableName); if (buffer == null) { - buffer = new IlpV4TableBuffer(tableName); + buffer = new QwpTableBuffer(tableName); tableBuffers.put(tableName, buffer); } currentTableBuffer = buffer; @@ -531,7 +531,7 @@ public void incrementPendingRowCount() { // ==================== Sender interface implementation ==================== @Override - public IlpV4WebSocketSender table(CharSequence tableName) { + public QwpWebSocketSender table(CharSequence tableName) { checkNotClosed(); // Fast path: if table name matches current, skip hashmap lookup if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { @@ -543,7 +543,7 @@ public IlpV4WebSocketSender table(CharSequence tableName) { currentTableName = tableName.toString(); currentTableBuffer = tableBuffers.get(currentTableName); if (currentTableBuffer == null) { - currentTableBuffer = new IlpV4TableBuffer(currentTableName); + currentTableBuffer = new QwpTableBuffer(currentTableName); tableBuffers.put(currentTableName, currentTableBuffer); } // Both modes accumulate rows until flush @@ -551,10 +551,10 @@ public IlpV4WebSocketSender table(CharSequence tableName) { } @Override - public IlpV4WebSocketSender symbol(CharSequence columnName, CharSequence value) { + public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); if (value != null) { // Register symbol in global dictionary and track max ID for delta calculation @@ -572,19 +572,19 @@ public IlpV4WebSocketSender symbol(CharSequence columnName, CharSequence value) } @Override - public IlpV4WebSocketSender boolColumn(CharSequence columnName, boolean value) { + public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); col.addBoolean(value); return this; } @Override - public IlpV4WebSocketSender longColumn(CharSequence columnName, long value) { + public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); col.addLong(value); return this; } @@ -596,28 +596,28 @@ public IlpV4WebSocketSender longColumn(CharSequence columnName, long value) { * @param value the int value * @return this sender for method chaining */ - public IlpV4WebSocketSender intColumn(CharSequence columnName, int value) { + public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); col.addInt(value); return this; } @Override - public IlpV4WebSocketSender doubleColumn(CharSequence columnName, double value) { + public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); col.addDouble(value); return this; } @Override - public IlpV4WebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); col.addString(value != null ? value.toString() : null); return this; } @@ -629,10 +629,10 @@ public IlpV4WebSocketSender stringColumn(CharSequence columnName, CharSequence v * @param value the short value * @return this sender for method chaining */ - public IlpV4WebSocketSender shortColumn(CharSequence columnName, short value) { + public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); col.addShort(value); return this; } @@ -646,10 +646,10 @@ public IlpV4WebSocketSender shortColumn(CharSequence columnName, short value) { * @param value the character value * @return this sender for method chaining */ - public IlpV4WebSocketSender charColumn(CharSequence columnName, char value) { + public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); col.addShort((short) value); return this; } @@ -662,10 +662,10 @@ public IlpV4WebSocketSender charColumn(CharSequence columnName, char value) { * @param hi the high 64 bits of the UUID * @return this sender for method chaining */ - public IlpV4WebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { + public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); col.addUuid(hi, lo); return this; } @@ -680,35 +680,35 @@ public IlpV4WebSocketSender uuidColumn(CharSequence columnName, long lo, long hi * @param l3 the most significant 64 bits * @return this sender for method chaining */ - public IlpV4WebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { + public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG256, true); col.addLong256(l0, l1, l2, l3); return this; } @Override - public IlpV4WebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { + public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); if (unit == ChronoUnit.NANOS) { - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); col.addLong(value); } else { long micros = toMicros(value, unit); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); col.addLong(micros); } return this; } @Override - public IlpV4WebSocketSender timestampColumn(CharSequence columnName, Instant value) { + public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value) { checkNotClosed(); checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); col.addLong(micros); return this; } @@ -832,7 +832,7 @@ private void flushPendingRows() { if (tableName == null) { continue; // Skip null entries (shouldn't happen but be safe) } - IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); if (tableBuffer == null) { continue; } @@ -859,7 +859,7 @@ private void flushPendingRows() { if (!useSchemaRef) { sentSchemaHashes.add(schemaKey); } - IlpBufferWriter buffer = encoder.getBuffer(); + QwpBufferWriter buffer = encoder.getBuffer(); // Copy to microbatch buffer and seal immediately // Each ILP v4 message must be in its own WebSocket frame @@ -1031,7 +1031,7 @@ private void flushSync() { if (tableName == null) { continue; } - IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); if (tableBuffer == null || tableBuffer.getRowCount() == 0) { continue; } @@ -1057,7 +1057,7 @@ private void flushSync() { } if (messageSize > 0) { - IlpBufferWriter buffer = encoder.getBuffer(); + QwpBufferWriter buffer = encoder.getBuffer(); // Track batch in InFlightWindow before sending long batchSequence = nextBatchSequence++; @@ -1192,7 +1192,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -1202,7 +1202,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -1212,7 +1212,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -1222,7 +1222,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(array); return this; } @@ -1232,7 +1232,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -1242,7 +1242,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -1252,7 +1252,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -1262,7 +1262,7 @@ public Sender longArray(CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(array); return this; } @@ -1274,7 +1274,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); col.addDecimal64(value); return this; } @@ -1284,7 +1284,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); col.addDecimal128(value); return this; } @@ -1294,7 +1294,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); col.addDecimal256(value); return this; } @@ -1307,7 +1307,7 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { try { java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); Decimal256 decimal = Decimal256.fromBigDecimal(bd); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); col.addDecimal256(decimal); } catch (Exception e) { throw new LineSenderException("Failed to parse decimal value: " + value, e); @@ -1392,7 +1392,7 @@ public void close() { encoder.close(); tableBuffers.clear(); - LOG.info("IlpV4WebSocketSender closed"); + LOG.info("QwpWebSocketSender closed"); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java index dca05f2..2b78429 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.line.LineSenderException; import org.slf4j.Logger; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java index f5be4a4..8774ac2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java @@ -22,13 +22,13 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 2c29baa..35a5f77 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.std.Unsafe; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index b34926e..e47c1d8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index 158ec3d..bd99092 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -36,14 +36,14 @@ *

* Usage pattern: * 

- * IlpV4BitReader reader = new IlpV4BitReader();
+ * QwpBitReader reader = new QwpBitReader();
  * reader.reset(address, length);
  * int bit = reader.readBit();
  * long value = reader.readBits(numBits);
  * long signedValue = reader.readSigned(numBits);
  *
*/ -public class IlpV4BitReader { +public class QwpBitReader { private long startAddress; private long currentAddress; @@ -61,7 +61,7 @@ public class IlpV4BitReader { /** * Creates a new bit reader. Call {@link #reset} before use. */ - public IlpV4BitReader() { + public QwpBitReader() { } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java similarity index 97% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index 8be1600..624b083 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -37,7 +37,7 @@ *

* Usage pattern: * 

- * IlpV4BitWriter writer = new IlpV4BitWriter();
+ * QwpBitWriter writer = new QwpBitWriter();
  * writer.reset(address, capacity);
  * writer.writeBits(value, numBits);
  * writer.writeBits(value2, numBits2);
@@ -45,7 +45,7 @@
  * long bytesWritten = writer.getPosition() - address;
  *
*/ -public class IlpV4BitWriter { +public class QwpBitWriter { private long startAddress; private long currentAddress; @@ -59,7 +59,7 @@ public class IlpV4BitWriter { /** * Creates a new bit writer. Call {@link #reset} before use. */ - public IlpV4BitWriter() { + public QwpBitWriter() { } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java similarity index 89% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index cea87af..b9d9a26 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -22,16 +22,16 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; -import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * Represents a column definition in an ILP v4 schema. *

* This class is immutable and safe for caching. */ -public final class IlpV4ColumnDef { +public final class QwpColumnDef { private final String name; private final byte typeCode; private final boolean nullable; @@ -42,7 +42,7 @@ public final class IlpV4ColumnDef { * @param name the column name (UTF-8) * @param typeCode the ILP v4 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable) */ - public IlpV4ColumnDef(String name, byte typeCode) { + public QwpColumnDef(String name, byte typeCode) { this.name = name; // Extract nullable flag (high bit) and base type this.nullable = (typeCode & 0x80) != 0; @@ -56,7 +56,7 @@ public IlpV4ColumnDef(String name, byte typeCode) { * @param typeCode the base type code (0x01-0x0F) * @param nullable whether the column is nullable */ - public IlpV4ColumnDef(String name, byte typeCode, boolean nullable) { + public QwpColumnDef(String name, byte typeCode, boolean nullable) { this.name = name; this.typeCode = (byte) (typeCode & 0x7F); this.nullable = nullable; @@ -98,7 +98,7 @@ public boolean isNullable() { * Returns true if this is a fixed-width type. */ public boolean isFixedWidth() { - return IlpV4Constants.isFixedWidthType(typeCode); + return QwpConstants.isFixedWidthType(typeCode); } /** @@ -107,14 +107,14 @@ public boolean isFixedWidth() { * @return width in bytes, or -1 for variable-width types */ public int getFixedWidth() { - return IlpV4Constants.getFixedTypeSize(typeCode); + return QwpConstants.getFixedTypeSize(typeCode); } /** * Gets the type name for display purposes. */ public String getTypeName() { - return IlpV4Constants.getTypeName(typeCode); + return QwpConstants.getTypeName(typeCode); } /** @@ -137,7 +137,7 @@ public void validate() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - IlpV4ColumnDef that = (IlpV4ColumnDef) o; + QwpColumnDef that = (QwpColumnDef) o; return typeCode == that.typeCode && nullable == that.nullable && name.equals(that.name); diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 7b229f6..622dbdd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -22,12 +22,12 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; /** * Constants for the ILP v4 binary protocol. */ -public final class IlpV4Constants { +public final class QwpConstants { // ==================== Magic Bytes ==================== @@ -360,7 +360,7 @@ public final class IlpV4Constants { */ public static final int CAPABILITY_RESPONSE_SIZE = 8; - private IlpV4Constants() { + private QwpConstants() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java similarity index 97% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 84e4334..5281dbc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -44,7 +44,7 @@ * The encoder writes first two timestamps uncompressed, then encodes * remaining timestamps using delta-of-delta compression. */ -public class IlpV4GorillaEncoder { +public class QwpGorillaEncoder { private static final int BUCKET_12BIT_MAX = 2048; private static final int BUCKET_12BIT_MIN = -2047; @@ -53,12 +53,12 @@ public class IlpV4GorillaEncoder { private static final int BUCKET_7BIT_MIN = -63; private static final int BUCKET_9BIT_MAX = 256; private static final int BUCKET_9BIT_MIN = -255; - private final IlpV4BitWriter bitWriter = new IlpV4BitWriter(); + private final QwpBitWriter bitWriter = new QwpBitWriter(); /** * Creates a new Gorilla encoder. */ - public IlpV4GorillaEncoder() { + public QwpGorillaEncoder() { } /** @@ -166,7 +166,7 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps, return 0; } - int pos = 0; + int pos; // Write first timestamp uncompressed if (capacity < 8) { diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java index 738f2dd..90cf944 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -42,9 +42,9 @@ * Byte 1: 0b00000010 (bit 1 set, which is row 9) * */ -public final class IlpV4NullBitmap { +public final class QwpNullBitmap { - private IlpV4NullBitmap() { + private QwpNullBitmap() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 9996d7f..9edc3f6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -41,7 +41,7 @@ * * @see xxHash */ -public final class IlpV4SchemaHash { +public final class QwpSchemaHash { // XXHash64 constants private static final long PRIME64_1 = 0x9E3779B185EBCA87L; @@ -56,7 +56,7 @@ public final class IlpV4SchemaHash { // Thread-local Hasher to avoid allocation on every computeSchemaHash call private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); - private IlpV4SchemaHash() { + private QwpSchemaHash() { // utility class } @@ -340,13 +340,13 @@ public static long computeSchemaHash(DirectUtf8Sequence[] columnNames, byte[] co * @param columns list of column buffers * @return the schema hash */ - public static long computeSchemaHashDirect(io.questdb.client.std.ObjList columns) { + public static long computeSchemaHashDirect(io.questdb.client.std.ObjList columns) { // Use pooled hasher to avoid allocation Hasher hasher = HASHER_POOL.get(); hasher.reset(DEFAULT_SEED); for (int i = 0, n = columns.size(); i < n; i++) { - IlpV4TableBuffer.ColumnBuffer col = columns.get(i); + QwpTableBuffer.ColumnBuffer col = columns.get(i); String name = col.getName(); // Encode UTF-8 directly without allocating byte array for (int j = 0, len = name.length(); j < len; j++) { diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 6d1af59..68f75ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.ArrayBufferAppender; @@ -38,7 +38,7 @@ import java.util.Arrays; -import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * Buffers rows for a single table in columnar format. @@ -46,7 +46,7 @@ * This buffer accumulates row data column by column, allowing efficient * encoding to the ILP v4 wire format. */ -public class IlpV4TableBuffer { +public class QwpTableBuffer { private final String tableName; private final ObjList columns; @@ -56,10 +56,10 @@ public class IlpV4TableBuffer { private int rowCount; private long schemaHash; private boolean schemaHashComputed; - private IlpV4ColumnDef[] cachedColumnDefs; + private QwpColumnDef[] cachedColumnDefs; private boolean columnDefsCacheValid; - public IlpV4TableBuffer(String tableName) { + public QwpTableBuffer(String tableName) { this.tableName = tableName; this.columns = new ObjList<>(); this.columnNameToIndex = new CharSequenceIntHashMap(); @@ -100,12 +100,12 @@ public ColumnBuffer getColumn(int index) { /** * Returns the column definitions (cached for efficiency). */ - public IlpV4ColumnDef[] getColumnDefs() { + public QwpColumnDef[] getColumnDefs() { if (!columnDefsCacheValid || cachedColumnDefs == null || cachedColumnDefs.length != columns.size()) { - cachedColumnDefs = new IlpV4ColumnDef[columns.size()]; + cachedColumnDefs = new QwpColumnDef[columns.size()]; for (int i = 0; i < columns.size(); i++) { ColumnBuffer col = columns.get(i); - cachedColumnDefs[i] = new IlpV4ColumnDef(col.name, col.type, col.nullable); + cachedColumnDefs[i] = new QwpColumnDef(col.name, col.type, col.nullable); } columnDefsCacheValid = true; } @@ -204,14 +204,14 @@ public void cancelCurrentRow() { /** * Returns the schema hash for this table. *

- * The hash is computed to match what IlpV4Schema.computeSchemaHash() produces: + * The hash is computed to match what QwpSchema.computeSchemaHash() produces: * - Uses wire type codes (with nullable bit) * - Hash is over name bytes + type code for each column */ public long getSchemaHash() { if (!schemaHashComputed) { // Compute hash directly from column buffers without intermediate arrays - schemaHash = IlpV4SchemaHash.computeSchemaHashDirect(columns); + schemaHash = QwpSchemaHash.computeSchemaHashDirect(columns); schemaHashComputed = true; } return schemaHash; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java index cd150d4..3b98a70 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -38,7 +38,7 @@ *

* This implementation is designed for zero-allocation on hot paths. */ -public final class IlpV4Varint { +public final class QwpVarint { /** * Maximum number of bytes needed to encode a 64-bit varint. @@ -56,7 +56,7 @@ public final class IlpV4Varint { */ private static final int DATA_MASK = 0x7F; - private IlpV4Varint() { + private QwpVarint() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java similarity index 96% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java index b0542e2..44e596d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; /** * ZigZag encoding/decoding for signed integers. @@ -50,9 +50,9 @@ * negative numbers like -1 become small positive numbers (1), which * encode efficiently as varints. */ -public final class IlpV4ZigZag { +public final class QwpZigZag { - private IlpV4ZigZag() { + private QwpZigZag() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 2cb001c..629767f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; /** * WebSocket close status codes as defined in RFC 6455. diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index 53d02b3..fb980ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; import io.questdb.client.std.Unsafe; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index d94bcd5..e4d423b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; import io.questdb.client.std.Unsafe; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java index 5248942..0bbcddd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; import io.questdb.client.std.Unsafe; import io.questdb.client.std.str.Utf8Sequence; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java index 03fe188..40466ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; /** * WebSocket frame opcodes as defined in RFC 6455. diff --git a/core/src/main/java/module-info.java b/core/src/main/java/module-info.java index 45b319c..cf4c93b 100644 --- a/core/src/main/java/module-info.java +++ b/core/src/main/java/module-info.java @@ -56,7 +56,7 @@ exports io.questdb.client.cairo.arr; exports io.questdb.client.cutlass.line.array; exports io.questdb.client.cutlass.line.udp; - exports io.questdb.client.cutlass.ilpv4.client; - exports io.questdb.client.cutlass.ilpv4.protocol; - exports io.questdb.client.cutlass.ilpv4.websocket; + exports io.questdb.client.cutlass.qwp.client; + exports io.questdb.client.cutlass.qwp.protocol; + exports io.questdb.client.cutlass.qwp.websocket; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java new file mode 100644 index 0000000..023cf13 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -0,0 +1,832 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +/** + * Tests for InFlightWindow. + *

+ * The window assumes sequential batch IDs and cumulative acknowledgments. It + * tracks only the range [lastAcked+1, highestSent] rather than individual batch + * IDs. + */ +public class InFlightWindowTest { + + @Test + public void testBasicAddAndAcknowledge() { + InFlightWindow window = new InFlightWindow(8, 1000); + + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + + // Add a batch (sequential: 0) + window.addInFlight(0); + assertFalse(window.isEmpty()); + assertEquals(1, window.getInFlightCount()); + + // Acknowledge it (cumulative ACK up to 0) + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + assertEquals(1, window.getTotalAcked()); + } + + @Test + public void testMultipleBatches() { + InFlightWindow window = new InFlightWindow(8, 1000); + + // Add sequential batches 0-4 + for (long i = 0; i < 5; i++) { + window.addInFlight(i); + } + assertEquals(5, window.getInFlightCount()); + + // Cumulative ACK up to 2 (acknowledges 0, 1, 2) + assertEquals(3, window.acknowledgeUpTo(2)); + assertEquals(2, window.getInFlightCount()); + + // Cumulative ACK up to 4 (acknowledges 3, 4) + assertEquals(2, window.acknowledgeUpTo(4)); + assertTrue(window.isEmpty()); + assertEquals(5, window.getTotalAcked()); + } + + @Test + public void testAcknowledgeAlreadyAcked() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // ACK up to 1 + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + + // ACK for already acknowledged sequence returns true (idempotent) + assertTrue(window.acknowledge(0)); + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + } + + @Test + public void testWindowFull() { + InFlightWindow window = new InFlightWindow(3, 1000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + assertTrue(window.isFull()); + assertEquals(3, window.getInFlightCount()); + + // Free slots by ACKing + window.acknowledgeUpTo(1); + assertFalse(window.isFull()); + assertEquals(1, window.getInFlightCount()); + } + + @Test + public void testWindowBlocksWhenFull() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(2); + blocked.set(false); + finished.countDown(); + }); + addThread.start(); + + // Wait for thread to start and block + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Give time to block + assertTrue(blocked.get()); + + // Free a slot + window.acknowledge(0); + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testWindowBlocksTimeout() { + InFlightWindow window = new InFlightWindow(2, 100); // 100ms timeout + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + // Try to add another - should timeout + long start = System.currentTimeMillis(); + try { + window.addInFlight(2); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); + } + + @Test + public void testAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertTrue(waiting.get()); + + // Cumulative ACK all batches + window.acknowledgeUpTo(2); + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + } + + @Test + public void testAwaitEmptyTimeout() { + InFlightWindow window = new InFlightWindow(8, 100); // 100ms timeout + + window.addInFlight(0); + + long start = System.currentTimeMillis(); + try { + window.awaitEmpty(); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); + } + + @Test + public void testAwaitEmptyAlreadyEmpty() { + InFlightWindow window = new InFlightWindow(8, 1000); + + // Should return immediately + window.awaitEmpty(); + assertTrue(window.isEmpty()); + } + + @Test + public void testFailBatch() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // Fail batch 0 + RuntimeException error = new RuntimeException("Test error"); + window.fail(0, error); + + assertEquals(1, window.getTotalFailed()); + assertNotNull(window.getLastError()); + } + + @Test + public void testFailPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Test error")); + + // Subsequent operations should throw + try { + window.addInFlight(1); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + try { + window.awaitEmpty(); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + } + + @Test + public void testFailAllPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.failAll(new RuntimeException("Transport down")); + + try { + window.awaitEmpty(); + fail("Expected exception due to failAll"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + assertTrue(e.getMessage().contains("Transport down")); + } + } + + @Test + public void testClearError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Test error")); + + assertNotNull(window.getLastError()); + + window.clearError(); + assertNull(window.getLastError()); + + // Should work again + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); // 0 and 1 both in window (fail doesn't remove) + } + + @Test + public void testReset() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.fail(2, new RuntimeException("Test")); + + window.reset(); + + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + assertEquals(0, window.getInFlightCount()); + } + + @Test + public void testConcurrentAddAndAck() throws Exception { + InFlightWindow window = new InFlightWindow(4, 5000); + int numOperations = 100; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numOperations; i++) { + window.addInFlight(i); + highestAdded.set(i); + Thread.sleep(1); // Small delay + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // ACK thread (cumulative ACKs) + Thread acker = new Thread(() -> { + try { + Thread.sleep(10); // Let sender get ahead + int lastAcked = -1; + while (lastAcked < numOperations - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } + Thread.sleep(1); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(10, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numOperations, window.getTotalAcked()); + } + + @Test + public void testFailWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); + + // Thread that will block on add + Thread addThread = new Thread(() -> { + started.countDown(); + try { + window.addInFlight(2); + } catch (LineSenderException e) { + caught.set(e); + } + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Let it block + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + addThread.join(1000); + assertFalse(addThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); + } + + @Test + public void testFailWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5000); + + window.addInFlight(0); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); + + // Thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + try { + window.awaitEmpty(); + } catch (LineSenderException e) { + caught.set(e); + } + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Let it block + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + waitThread.join(1000); + assertFalse(waitThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidWindowSize() { + new InFlightWindow(0, 1000); + } + + @Test + public void testGetMaxWindowSize() { + InFlightWindow window = new InFlightWindow(16, 1000); + assertEquals(16, window.getMaxWindowSize()); + } + + @Test + public void testRapidAddAndAck() { + InFlightWindow window = new InFlightWindow(16, 5000); + + // Rapid add and ack in same thread + for (int i = 0; i < 10000; i++) { + window.addInFlight(i); + assertTrue(window.acknowledge(i)); + } + + assertTrue(window.isEmpty()); + assertEquals(10000, window.getTotalAcked()); + } + + @Test + public void testFillAndDrainRepeatedly() { + InFlightWindow window = new InFlightWindow(4, 1000); + + int batchId = 0; + for (int cycle = 0; cycle < 100; cycle++) { + // Fill + int startBatch = batchId; + for (int i = 0; i < 4; i++) { + window.addInFlight(batchId++); + } + assertTrue(window.isFull()); + assertEquals(4, window.getInFlightCount()); + + // Drain with cumulative ACK + window.acknowledgeUpTo(batchId - 1); + assertTrue(window.isEmpty()); + } + + assertEquals(400, window.getTotalAcked()); + } + + @Test + public void testMultipleResets() { + InFlightWindow window = new InFlightWindow(8, 1000); + + for (int cycle = 0; cycle < 10; cycle++) { + window.addInFlight(cycle); + window.reset(); + + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + } + } + + @Test + public void testFailThenClearThenAdd() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Error")); + + // Should not be able to add + try { + window.addInFlight(1); + fail("Expected exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + // Clear error + window.clearError(); + + // Should work now + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testDefaultWindowSize() { + InFlightWindow window = new InFlightWindow(); + assertEquals(InFlightWindow.DEFAULT_WINDOW_SIZE, window.getMaxWindowSize()); + } + + @Test + public void testSmallestPossibleWindow() { + InFlightWindow window = new InFlightWindow(1, 1000); + + window.addInFlight(0); + assertTrue(window.isFull()); + + window.acknowledge(0); + assertFalse(window.isFull()); + } + + @Test + public void testVeryLargeWindow() { + InFlightWindow window = new InFlightWindow(10000, 1000); + + // Add many batches + for (int i = 0; i < 5000; i++) { + window.addInFlight(i); + } + assertEquals(5000, window.getInFlightCount()); + assertFalse(window.isFull()); + + // ACK half + window.acknowledgeUpTo(2499); + assertEquals(2500, window.getInFlightCount()); + } + + @Test + public void testZeroBatchId() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + assertEquals(1, window.getInFlightCount()); + + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + } + + // ==================== CUMULATIVE ACK TESTS ==================== + + @Test + public void testAcknowledgeUpToBasic() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches 0-9 + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + assertEquals(10, window.getInFlightCount()); + + // ACK up to 5 (should remove 0-5, leaving 6-9) + int acked = window.acknowledgeUpTo(5); + assertEquals(6, acked); + assertEquals(4, window.getInFlightCount()); + assertEquals(6, window.getTotalAcked()); + } + + @Test + public void testAcknowledgeUpToIdempotent() { + InFlightWindow window = new InFlightWindow(16, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + // First ACK + assertEquals(3, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // Duplicate ACK - should be no-op + assertEquals(0, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // ACK with lower sequence - should be no-op + assertEquals(0, window.acknowledgeUpTo(1)); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(3, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + assertTrue(window.isFull()); + + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(3); + blocked.set(false); + finished.countDown(); + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Give time to block + assertTrue(blocked.get()); + + // Cumulative ACK frees multiple slots + window.acknowledgeUpTo(1); // Removes 0 and 1 + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); // batch 2 and 3 + } + + @Test + public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(16, 5000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertTrue(waiting.get()); + + // Single cumulative ACK clears all + window.acknowledgeUpTo(2); + + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToEmpty() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // ACK on empty window should be no-op + assertEquals(0, window.acknowledgeUpTo(100)); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToAllBatches() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + + // ACK all with high sequence + int acked = window.acknowledgeUpTo(Long.MAX_VALUE); + assertEquals(10, acked); + assertTrue(window.isEmpty()); + } + + @Test + public void testConcurrentAddAndCumulativeAck() throws Exception { + InFlightWindow window = new InFlightWindow(100, 10000); + int numBatches = 500; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // ACK thread using cumulative ACKs + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } else { + Thread.sleep(1); + } + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(30, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } + + @Test + public void testTryAddInFlight() { + InFlightWindow window = new InFlightWindow(2, 1000); + + // Should succeed + assertTrue(window.tryAddInFlight(0)); + assertTrue(window.tryAddInFlight(1)); + + // Should fail - window full + assertFalse(window.tryAddInFlight(2)); + + // After ACK, should succeed + window.acknowledge(0); + assertTrue(window.tryAddInFlight(2)); + } + + @Test + public void testHasWindowSpace() { + InFlightWindow window = new InFlightWindow(2, 1000); + + assertTrue(window.hasWindowSpace()); + window.addInFlight(0); + assertTrue(window.hasWindowSpace()); + window.addInFlight(1); + assertFalse(window.hasWindowSpace()); + + window.acknowledge(0); + assertTrue(window.hasWindowSpace()); + } + + @Test + public void testHighConcurrencyStress() throws Exception { + InFlightWindow window = new InFlightWindow(8, 30000); + int numBatches = 10000; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Fast sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // Fast ACK thread + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } + // No sleep - maximum contention + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(60, TimeUnit.SECONDS)); + if (error.get() != null) { + error.get().printStackTrace(); + } + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } +} From 10c306ab596c58c1b72637fc0ab622cd52d62335 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 23:02:52 +0000 Subject: [PATCH 006/230] tidy --- .../client/test/cutlass/line/LineSenderBuilderTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java index bf9f5c7..e684e52 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -338,7 +338,7 @@ public void testMalformedPortInAddress() throws Exception { @Test public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]", - Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(65535))); + () -> Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(65535))); } @Test @@ -350,13 +350,13 @@ public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws E @Test public void testMaxRetriesNotSupportedForTcp() throws Exception { assertMemoryLeak(() -> assertThrows("retrying is not supported for TCP protocol", - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100))); + () -> Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100))); } @Test public void testMinRequestThroughputCannotBeNegative() throws Exception { assertMemoryLeak(() -> assertThrows("minimum request throughput must not be negative [minRequestThroughput=-100]", - Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100))); + () -> Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100))); } @Test From f12a6f99a477cea7b8ff3a84beff428ce7b6c416 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 00:29:09 +0000 Subject: [PATCH 007/230] wip --- .../qwp/client/QwpWebSocketSender.java | 17 +- .../questdb/client/test/AbstractQdbTest.java | 2 +- .../LineSenderBuilderWebSocketTest.java | 2 +- .../cutlass/qwp/client/QwpSenderTest.java | 952 ++++++++++++++++++ core/src/test/java/module-info.java | 1 + 5 files changed, 971 insertions(+), 3 deletions(-) rename core/src/test/java/io/questdb/client/test/cutlass/{line => qwp/client}/LineSenderBuilderWebSocketTest.java (99%) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 6b0343f..4a70fb1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -613,6 +613,21 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { return this; } + /** + * Adds a FLOAT column value to the current row. + * + * @param columnName the column name + * @param value the float value + * @return this sender for method chaining + */ + public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_FLOAT, false); + col.addFloat(value); + return this; + } + @Override public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); @@ -1258,7 +1273,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { } @Override - public Sender longArray(CharSequence name, LongArray array) { + public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java index 60f3ed2..81242bd 100644 --- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java +++ b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java @@ -426,7 +426,7 @@ protected static String getPgUser() { * Get whether a QuestDB instance is running locally. */ protected static boolean getQuestDBRunning() { - return getConfigBool("QUESTDB_RUNNING", "questdb.running", false); + return getConfigBool("QUESTDB_RUNNING", "questdb.running", true); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java similarity index 99% rename from core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java rename to core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 92c79d0..ee80665 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.test.cutlass.line; +package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java new file mode 100644 index 0000000..4867640 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -0,0 +1,952 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.std.Decimal256; +import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +/** + * Integration tests for the QWP (QuestDB Wire Protocol) WebSocket sender. + *

+ * Tests verify that all QWP native types arrive correctly (exact type match) + * and that reasonable type coercions work (e.g., client sends INT but server + * column is LONG). + *

+ * Tests are skipped if no QuestDB instance is running + * ({@code -Dquestdb.running=true}). + */ +public class QwpSenderTest extends AbstractLineSenderTest { + + @BeforeClass + public static void setUpStatic() { + AbstractLineSenderTest.setUpStatic(); + } + + // === Exact Type Match Tests === + + @Test + public void testBoolean() throws Exception { + String table = "test_qwp_boolean"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("b", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .boolColumn("b", false) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\ttimestamp\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testByte() throws Exception { + String table = "test_qwp_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("b", (short) -1) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testChar() throws Exception { + String table = "test_qwp_char"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("c", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '\u00FC') // ü + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '\u4E2D') // 中 + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "c\ttimestamp\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "\u00FC\t1970-01-01T00:00:02.000000000Z\n" + + "\u4E2D\t1970-01-01T00:00:03.000000000Z\n", + "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testDecimal() throws Exception { + String table = "test_qwp_decimal"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "0.01") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(42_000, 3)) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + } + + @Test + public void testDouble() throws Exception { + String table = "test_qwp_double"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("d", 42.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", -1.0E10) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MIN_VALUE) + .at(4_000_000, ChronoUnit.MICROS); + // NaN and Inf should be stored as null + sender.table(table) + .doubleColumn("d", Double.NaN) + .at(5_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.POSITIVE_INFINITY) + .at(6_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.NEGATIVE_INFINITY) + .at(7_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 7); + assertSqlEventually( + "d\ttimestamp\n" + + "42.5\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + + "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + + "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + + "null\t1970-01-01T00:00:05.000000000Z\n" + + "null\t1970-01-01T00:00:06.000000000Z\n" + + "null\t1970-01-01T00:00:07.000000000Z\n", + "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testDoubleArray() throws Exception { + String table = "test_qwp_double_array"; + useTable(table); + + double[] arr1d = createDoubleArray(5); + double[][] arr2d = createDoubleArray(2, 3); + double[][][] arr3d = createDoubleArray(1, 2, 3); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("a1", arr1d) + .doubleArray("a2", arr2d) + .doubleArray("a3", arr3d) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testDoubleToDecimal() throws Exception { + String table = "test_qwp_double_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("d", 123.45) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", -42.10) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToFloat() throws Exception { + String table = "test_qwp_double_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("f", 1.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("f", -42.25) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testFloat() throws Exception { + String table = "test_qwp_float"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("f", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("d", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testInt() throws Exception { + String table = "test_qwp_int"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Integer.MIN_VALUE is the null sentinel for INT + sender.table(table) + .intColumn("i", Integer.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", -42) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "i\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("l", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", Integer.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong() throws Exception { + String table = "test_qwp_long"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG + sender.table(table) + .longColumn("l", Long.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testLong256() throws Exception { + String table = "test_qwp_long256"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros + sender.table(table) + .long256Column("v", 0, 0, 0, 0) + .at(1_000_000, ChronoUnit.MICROS); + // Mixed values + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in INT range should succeed + sender.table(table) + .longColumn("i", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("i", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testMultipleRowsAndBatching() throws Exception { + String table = "test_qwp_multiple_rows"; + useTable(table); + + int rowCount = 1000; + try (QwpWebSocketSender sender = createQwpSender()) { + for (int i = 0; i < rowCount; i++) { + sender.table(table) + .symbol("sym", "s" + (i % 10)) + .longColumn("val", i) + .doubleColumn("dbl", i * 1.5) + .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); + } + sender.flush(); + } + + assertTableSizeEventually(table, rowCount); + } + + @Test + public void testShort() throws Exception { + String table = "test_qwp_short"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Short.MIN_VALUE is the null sentinel for SHORT + sender.table(table) + .shortColumn("s", Short.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testShortToInt() throws Exception { + String table = "test_qwp_short_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("i", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("i", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToLong() throws Exception { + String table = "test_qwp_short_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("l", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("l", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testString() throws Exception { + String table = "test_qwp_string"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "hello world") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "non-ascii \u00E4\u00F6\u00FC") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "s\ttimestamp\n" + + "hello world\t1970-01-01T00:00:01.000000000Z\n" + + "non-ascii \u00E4\u00F6\u00FC\t1970-01-01T00:00:02.000000000Z\n" + + "\t1970-01-01T00:00:03.000000000Z\n" + + "null\t1970-01-01T00:00:04.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testStringToChar() throws Exception { + String table = "test_qwp_string_to_char"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("c", "A") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("c", "Hello") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "c\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "H\t1970-01-01T00:00:02.000000000Z\n", + "SELECT c, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToSymbol() throws Exception { + String table = "test_qwp_string_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToUuid() throws Exception { + String table = "test_qwp_string_to_uuid"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "u\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT u, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testSymbol() throws Exception { + String table = "test_qwp_symbol"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "alpha") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "beta") + .at(2_000_000, ChronoUnit.MICROS); + // repeated value reuses dictionary entry + sender.table(table) + .symbol("s", "alpha") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\ttimestamp\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "beta\t1970-01-01T00:00:02.000000000Z\n" + + "alpha\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testTimestampMicros() throws Exception { + String table = "test_qwp_timestamp_micros"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + sender.table(table) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\ttimestamp\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, timestamp FROM " + table); + } + + @Test + public void testTimestampMicrosToNanos() throws Exception { + String table = "test_qwp_timestamp_micros_to_nanos"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP WITH TIME ZONE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + sender.table(table) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testTimestampNanos() throws Exception { + String table = "test_qwp_timestamp_nanos"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_000_000_000L; // 2022-02-25T00:00:00Z in nanos + sender.table(table) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .at(tsNanos, ChronoUnit.NANOS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testTimestampNanosToMicros() throws Exception { + String table = "test_qwp_timestamp_nanos_to_micros"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_123_456_789L; + sender.table(table) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + // Nanoseconds truncated to microseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); + } + + @Test + public void testUuid() throws Exception { + String table = "test_qwp_uuid"; + useTable(table); + + UUID uuid1 = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("u", uuid1.getLeastSignificantBits(), uuid1.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .uuidColumn("u", uuid2.getLeastSignificantBits(), uuid2.getMostSignificantBits()) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "u\ttimestamp\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + + "11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z\n", + "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testWriteAllTypesInOneRow() throws Exception { + String table = "test_qwp_all_types"; + useTable(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + double[] arr1d = {1.0, 2.0, 3.0}; + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("sym", "test_symbol") + .boolColumn("bool_col", true) + .shortColumn("short_col", (short) 42) + .intColumn("int_col", 100_000) + .longColumn("long_col", 1_000_000_000L) + .floatColumn("float_col", 2.5f) + .doubleColumn("double_col", 3.14) + .stringColumn("string_col", "hello") + .charColumn("char_col", 'Z') + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .uuidColumn("uuid_col", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .long256Column("long256_col", 1, 0, 0, 0) + .doubleArray("arr_col", arr1d) + .decimalColumn("decimal_col", "99.99") + .at(tsMicros, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + // === Helper Methods === + + private QwpWebSocketSender createQwpSender() { + return QwpWebSocketSender.connect(getQuestDbHost(), getHttpPort()); + } +} diff --git a/core/src/test/java/module-info.java b/core/src/test/java/module-info.java index 86341d8..7e39674 100644 --- a/core/src/test/java/module-info.java +++ b/core/src/test/java/module-info.java @@ -36,4 +36,5 @@ exports io.questdb.client.test; exports io.questdb.client.test.cairo; exports io.questdb.client.test.cutlass.line; + exports io.questdb.client.test.cutlass.qwp.client; } From d442198405b95e724c0fd6b1ebf123cfc741fcac Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 01:24:18 +0000 Subject: [PATCH 008/230] wip2 --- .../cutlass/qwp/client/QwpSenderTest.java | 480 +++++++++++++++++- 1 file changed, 474 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index 4867640..cbb5c8d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -111,10 +111,10 @@ public void testChar() throws Exception { .charColumn("c", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", '\u00FC') // ü + .charColumn("c", 'ü') // ü .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", '\u4E2D') // 中 + .charColumn("c", '中') // 中 .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -123,8 +123,8 @@ public void testChar() throws Exception { assertSqlEventually( "c\ttimestamp\n" + "A\t1970-01-01T00:00:01.000000000Z\n" + - "\u00FC\t1970-01-01T00:00:02.000000000Z\n" + - "\u4E2D\t1970-01-01T00:00:03.000000000Z\n", + "ü\t1970-01-01T00:00:02.000000000Z\n" + + "中\t1970-01-01T00:00:03.000000000Z\n", "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -377,6 +377,166 @@ public void testIntToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", Integer.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testIntToDouble() throws Exception { String table = "test_qwp_int_to_double"; @@ -513,6 +673,146 @@ public void testLongToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 1_000_000_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", Long.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testLongToInt() throws Exception { String table = "test_qwp_long_to_int"; @@ -608,6 +908,174 @@ public void testShort() throws Exception { assertTableSizeEventually(table, 3); } + @Test + public void testShortToDecimal128() throws Exception { + String table = "test_qwp_short_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", Short.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", Short.MIN_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "32767.00\t1970-01-01T00:00:01.000000000Z\n" + + "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal16() throws Exception { + String table = "test_qwp_short_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal256() throws Exception { + String table = "test_qwp_short_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal32() throws Exception { + String table = "test_qwp_short_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal64() throws Exception { + String table = "test_qwp_short_to_decimal64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal8() throws Exception { + String table = "test_qwp_short_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testShortToInt() throws Exception { String table = "test_qwp_short_to_int"; @@ -674,7 +1142,7 @@ public void testString() throws Exception { .stringColumn("s", "hello world") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "non-ascii \u00E4\u00F6\u00FC") + .stringColumn("s", "non-ascii äöü") .at(2_000_000, ChronoUnit.MICROS); sender.table(table) .stringColumn("s", "") @@ -689,7 +1157,7 @@ public void testString() throws Exception { assertSqlEventually( "s\ttimestamp\n" + "hello world\t1970-01-01T00:00:01.000000000Z\n" + - "non-ascii \u00E4\u00F6\u00FC\t1970-01-01T00:00:02.000000000Z\n" + + "non-ascii äöü\t1970-01-01T00:00:02.000000000Z\n" + "\t1970-01-01T00:00:03.000000000Z\n" + "null\t1970-01-01T00:00:04.000000000Z\n", "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); From ea798b80124bf28fb64c7bc1c1523f3f00ac869d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 03:14:19 +0000 Subject: [PATCH 009/230] wip3 --- .../cutlass/qwp/client/QwpSenderTest.java | 1781 +++++++++++++++-- 1 file changed, 1564 insertions(+), 217 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index cbb5c8d..cf73ee3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -24,9 +24,11 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.Decimal256; import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -245,126 +247,1448 @@ public void testDoubleToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testDoubleToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_decimal_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("d", 123.456) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") + ); + } + } + + @Test + public void testDoubleToByte() throws Exception { + String table = "test_qwp_double_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("b", 42.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("b", -100.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToBytePrecisionLossError() throws Exception { + String table = "test_qwp_double_to_byte_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("b", 42.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("42.5") + ); + } + } + + @Test + public void testDoubleToByteOverflowError() throws Exception { + String table = "test_qwp_double_to_byte_ovf"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("b", 200.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 200 out of range for BYTE") + ); + } + } + @Test public void testDoubleToFloat() throws Exception { String table = "test_qwp_double_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("f", 1.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("f", -42.25) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testDoubleToInt() throws Exception { + String table = "test_qwp_double_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("i", 100_000.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("i", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "100000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToIntPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_int_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("i", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("3.14") + ); + } + } + + @Test + public void testDoubleToLong() throws Exception { + String table = "test_qwp_double_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("l", 1_000_000.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("l", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "1000000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToString() throws Exception { + String table = "test_qwp_double_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("s", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("s", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToSymbol() throws Exception { + String table = "test_qwp_double_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "sym SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("sym", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "sym\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToVarchar() throws Exception { + String table = "test_qwp_double_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("v", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloat() throws Exception { + String table = "test_qwp_float"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("f", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testFloatToDecimal() throws Exception { + String table = "test_qwp_float_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("d", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.50\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_decimal_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.25f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("scale=1") + ); + } + } + + @Test + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("d", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToInt() throws Exception { + String table = "test_qwp_float_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("i", 42.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("i", -100.0f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToIntPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_int_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("i", 3.14f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") + ); + } + } + + @Test + public void testFloatToLong() throws Exception { + String table = "test_qwp_float_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("l", 1000.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "l\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToString() throws Exception { + String table = "test_qwp_float_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("s", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToSymbol() throws Exception { + String table = "test_qwp_float_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "sym SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("sym", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "sym\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToVarchar() throws Exception { + String table = "test_qwp_float_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testInt() throws Exception { + String table = "test_qwp_int"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Integer.MIN_VALUE is the null sentinel for INT + sender.table(table) + .intColumn("i", Integer.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", -42) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "i\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testIntToBooleanCoercionError() throws Exception { + String table = "test_qwp_int_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("b", 1) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and BOOLEAN but got: " + msg, + msg.contains("INT") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testIntToByte() throws Exception { + String table = "test_qwp_int_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("b", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("b", -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToByteOverflowError() throws Exception { + String table = "test_qwp_int_to_byte_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("b", 128) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); + } + } + + @Test + public void testIntToCharCoercionError() throws Exception { + String table = "test_qwp_int_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("c", 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and CHAR but got: " + msg, + msg.contains("INT") && msg.contains("CHAR") + ); + } + } + + @Test + public void testIntToDate() throws Exception { + String table = "test_qwp_int_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // 86_400_000 millis = 1 day + sender.table(table) + .intColumn("d", 86_400_000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", Integer.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToFloat() throws Exception { + String table = "test_qwp_int_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("f", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToGeoHashCoercionError() throws Exception { + String table = "test_qwp_int_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("g", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning INT but got: " + msg, + msg.contains("type coercion from INT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("l", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", Integer.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToLong256CoercionError() throws Exception { + String table = "test_qwp_int_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to LONG256 is not supported") + ); + } + } + + @Test + public void testIntToShort() throws Exception { + String table = "test_qwp_int_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 1000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -32768) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 32767) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToShortOverflowError() throws Exception { + String table = "test_qwp_int_to_short_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 32768) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") + ); + } + } + + @Test + public void testIntToString() throws Exception { + String table = "test_qwp_int_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("f", 1.5) + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("f", -42.25) + .intColumn("s", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloat() throws Exception { - String table = "test_qwp_float"; + public void testIntToSymbol() throws Exception { + String table = "test_qwp_int_to_symbol"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("f", 1.5f) + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", -42.25f) + .intColumn("s", -1) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", 0.0f) + .intColumn("s", 0) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDouble() throws Exception { - String table = "test_qwp_float_to_double"; + public void testIntToTimestamp() throws Exception { + String table = "test_qwp_int_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 1_000_000 micros = 1 second sender.table(table) - .floatColumn("d", 1.5f) + .intColumn("t", 1_000_000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .intColumn("t", 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testInt() throws Exception { - String table = "test_qwp_int"; + public void testIntToUuidCoercionError() throws Exception { + String table = "test_qwp_int_to_uuid_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .intColumn("i", Integer.MIN_VALUE) + .intColumn("u", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to UUID is not supported") + ); + } + } + + @Test + public void testIntToVarchar() throws Exception { + String table = "test_qwp_int_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("i", 0) + .intColumn("v", -100) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("i", Integer.MAX_VALUE) + .intColumn("v", Integer.MAX_VALUE) .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong() throws Exception { + String table = "test_qwp_long"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .intColumn("i", -42) - .at(4_000_000, ChronoUnit.MICROS); + .longColumn("l", Long.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\ttimestamp\n" + + "l\ttimestamp\n" + "null\t1970-01-01T00:00:01.000000000Z\n" + "0\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n" + - "-42\t1970-01-01T00:00:04.000000000Z\n", - "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testIntToDecimal() throws Exception { - String table = "test_qwp_int_to_decimal"; + public void testLong256() throws Exception { + String table = "test_qwp_long256"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros + sender.table(table) + .long256Column("v", 0, 0, 0, 0) + .at(1_000_000, ChronoUnit.MICROS); + // Mixed values + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToBooleanCoercionError() throws Exception { + String table = "test_qwp_long_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and BOOLEAN but got: " + msg, + msg.contains("LONG") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testLongToByte() throws Exception { + String table = "test_qwp_long_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", -100) + .longColumn("b", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToByteOverflowError() throws Exception { + String table = "test_qwp_long_to_byte_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 128) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); + } + } + + @Test + public void testLongToCharCoercionError() throws Exception { + String table = "test_qwp_long_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("c", 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and CHAR but got: " + msg, + msg.contains("LONG") && msg.contains("CHAR") + ); + } + } + + @Test + public void testLongToDate() throws Exception { + String table = "test_qwp_long_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 86_400_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", 0L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -378,8 +1702,8 @@ public void testIntToDecimal() throws Exception { } @Test - public void testIntToDecimal128() throws Exception { - String table = "test_qwp_int_to_decimal128"; + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(38, 2), " + @@ -389,29 +1713,25 @@ public void testIntToDecimal128() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", 1_000_000_000L) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -1_000_000_000L) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal16() throws Exception { - String table = "test_qwp_int_to_decimal16"; + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(4, 1), " + @@ -421,29 +1741,25 @@ public void testIntToDecimal16() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal256() throws Exception { - String table = "test_qwp_int_to_decimal256"; + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(76, 2), " + @@ -453,61 +1769,53 @@ public void testIntToDecimal256() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", Long.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -1_000_000_000_000L) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal64() throws Exception { - String table = "test_qwp_int_to_decimal64"; + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "d DECIMAL(6, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", Integer.MAX_VALUE) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal8() throws Exception { - String table = "test_qwp_int_to_decimal8"; + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(2, 1), " + @@ -517,29 +1825,25 @@ public void testIntToDecimal8() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 5) + .longColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -9) + .longColumn("d", -9) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + "-9.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDouble() throws Exception { - String table = "test_qwp_int_to_double"; + public void testLongToDouble() throws Exception { + String table = "test_qwp_long_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DOUBLE, " + @@ -549,10 +1853,10 @@ public void testIntToDouble() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -566,304 +1870,321 @@ public void testIntToDouble() throws Exception { } @Test - public void testIntToLong() throws Exception { - String table = "test_qwp_int_to_long"; + public void testLongToFloat() throws Exception { + String table = "test_qwp_long_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("l", 42) + .longColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("l", Integer.MAX_VALUE) + .longColumn("f", -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", -1) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "2147483647\t1970-01-01T00:00:02.000000000Z\n" + - "-1\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong() throws Exception { - String table = "test_qwp_long"; + public void testLongToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long_to_geohash_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .longColumn("l", Long.MIN_VALUE) + .longColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", Long.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning LONG but got: " + msg, + msg.contains("type coercion from LONG to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "l\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLong256() throws Exception { - String table = "test_qwp_long256"; + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // All zeros + // Value in INT range should succeed sender.table(table) - .long256Column("v", 0, 0, 0, 0) + .longColumn("i", 42) .at(1_000_000, ChronoUnit.MICROS); - // Mixed values sender.table(table) - .long256Column("v", 1, 2, 3, 4) + .longColumn("i", -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal() throws Exception { - String table = "test_qwp_long_to_decimal"; + public void testLongToIntOverflowError() throws Exception { + String table = "test_qwp_long_to_int_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .longColumn("i", (long) Integer.MAX_VALUE + 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 2147483648 out of range for INT") + ); } + } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + @Test + public void testLongToLong256CoercionError() throws Exception { + String table = "test_qwp_long_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to LONG256 is not supported") + ); + } } @Test - public void testLongToDecimal128() throws Exception { - String table = "test_qwp_long_to_decimal128"; + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed sender.table(table) - .longColumn("d", 1_000_000_000L) + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000L) + .longColumn("s", -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal16() throws Exception { - String table = "test_qwp_long_to_decimal16"; + public void testLongToShortOverflowError() throws Exception { + String table = "test_qwp_long_to_short_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .longColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal256() throws Exception { - String table = "test_qwp_long_to_decimal256"; + public void testLongToString() throws Exception { + String table = "test_qwp_long_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", Long.MAX_VALUE) + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000_000L) + .longColumn("s", Long.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal32() throws Exception { - String table = "test_qwp_long_to_decimal32"; + public void testLongToSymbol() throws Exception { + String table = "test_qwp_long_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .longColumn("s", -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal8() throws Exception { - String table = "test_qwp_long_to_decimal8"; + public void testLongToTimestamp() throws Exception { + String table = "test_qwp_long_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 5) + .longColumn("t", 1_000_000L) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -9) + .longColumn("t", 0L) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToInt() throws Exception { - String table = "test_qwp_long_to_int"; + public void testLongToUuidCoercionError() throws Exception { + String table = "test_qwp_long_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in INT range should succeed sender.table(table) - .longColumn("i", 42) + .longColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("i", -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to UUID is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToShort() throws Exception { - String table = "test_qwp_long_to_short"; + public void testLongToVarchar() throws Exception { + String table = "test_qwp_long_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in SHORT range should succeed sender.table(table) - .longColumn("s", 42) + .longColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", -1) + .longColumn("v", Long.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test @@ -1380,6 +2701,32 @@ public void testUuid() throws Exception { "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); } + @Test + public void testUuidToShortCoercionError() throws Exception { + String table = "test_qwp_uuid_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to SHORT is not supported") + ); + } + } + @Test public void testWriteAllTypesInOneRow() throws Exception { String table = "test_qwp_all_types"; From 3e444a04a63a03a5753861926b9b5d3cb5852e0c Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 03:41:14 +0000 Subject: [PATCH 010/230] wip4 --- .../qwp/client/QwpWebSocketSender.java | 15 + .../cutlass/qwp/protocol/QwpTableBuffer.java | 75 +- .../cutlass/qwp/client/QwpSenderTest.java | 3081 ++++++++++++----- 3 files changed, 2221 insertions(+), 950 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4a70fb1..d428dbb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -596,6 +596,21 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { * @param value the int value * @return this sender for method chaining */ + /** + * Adds a BYTE column value to the current row. + * + * @param columnName the column name + * @param value the byte value + * @return this sender for method chaining + */ + public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false); + col.addByte(value); + return this; + } + public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 68f75ec..f70dd98 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -299,6 +299,7 @@ public static class ColumnBuffer { // For Decimal128: two longs per value (128-bit unscaled: high, low) // For Decimal256: four longs per value (256-bit unscaled: hh, hl, lh, ll) private byte decimalScale = -1; // Shared scale for column (-1 = not set) + private final Decimal256 rescaleTemp = new Decimal256(); // Reusable temp for rescaling private long[] decimal64Values; // Decimal64: one long per value private long[] decimal128High; // Decimal128: high 64 bits private long[] decimal128Low; // Decimal128: low 64 bits @@ -722,10 +723,10 @@ public void addLong256(long l0, long l1, long l2, long l3) { /** * Adds a Decimal64 value. - * All values in a decimal column must share the same scale. + * If the value's scale differs from the column's established scale, + * the value is automatically rescaled to match. * * @param value the Decimal64 value to add - * @throws LineSenderException if the scale doesn't match previous values */ public void addDecimal64(Decimal64 value) { if (value == null || value.isNull()) { @@ -733,17 +734,26 @@ public void addDecimal64(Decimal64 value) { return; } ensureCapacity(); - validateAndSetScale((byte) value.getScale()); - decimal64Values[valueCount++] = value.getValue(); + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + decimal64Values[valueCount++] = value.getValue(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getValue()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + decimal64Values[valueCount++] = rescaleTemp.getLl(); + } else { + decimal64Values[valueCount++] = value.getValue(); + } size++; } /** * Adds a Decimal128 value. - * All values in a decimal column must share the same scale. + * If the value's scale differs from the column's established scale, + * the value is automatically rescaled to match. * * @param value the Decimal128 value to add - * @throws LineSenderException if the scale doesn't match previous values */ public void addDecimal128(Decimal128 value) { if (value == null || value.isNull()) { @@ -751,7 +761,18 @@ public void addDecimal128(Decimal128 value) { return; } ensureCapacity(); - validateAndSetScale((byte) value.getScale()); + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getHigh(), value.getLow()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + decimal128High[valueCount] = rescaleTemp.getLh(); + decimal128Low[valueCount] = rescaleTemp.getLl(); + valueCount++; + size++; + return; + } decimal128High[valueCount] = value.getHigh(); decimal128Low[valueCount] = value.getLow(); valueCount++; @@ -760,10 +781,10 @@ public void addDecimal128(Decimal128 value) { /** * Adds a Decimal256 value. - * All values in a decimal column must share the same scale. + * If the value's scale differs from the column's established scale, + * the value is automatically rescaled to match. * * @param value the Decimal256 value to add - * @throws LineSenderException if the scale doesn't match previous values */ public void addDecimal256(Decimal256 value) { if (value == null || value.isNull()) { @@ -771,32 +792,20 @@ public void addDecimal256(Decimal256 value) { return; } ensureCapacity(); - validateAndSetScale((byte) value.getScale()); - decimal256Hh[valueCount] = value.getHh(); - decimal256Hl[valueCount] = value.getHl(); - decimal256Lh[valueCount] = value.getLh(); - decimal256Ll[valueCount] = value.getLl(); - valueCount++; - size++; - } - - /** - * Validates that the given scale matches the column's scale. - * If this is the first value, sets the column scale. - * - * @param scale the scale of the value being added - * @throws LineSenderException if the scale doesn't match - */ - private void validateAndSetScale(byte scale) { + Decimal256 src = value; if (decimalScale == -1) { - decimalScale = scale; - } else if (decimalScale != scale) { - throw new LineSenderException( - "decimal scale mismatch in column '" + name + "': expected " + - decimalScale + " but got " + scale + - ". All values in a decimal column must have the same scale." - ); + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.copyFrom(value); + rescaleTemp.rescale(decimalScale); + src = rescaleTemp; } + decimal256Hh[valueCount] = src.getHh(); + decimal256Hl[valueCount] = src.getHl(); + decimal256Lh[valueCount] = src.getLh(); + decimal256Ll[valueCount] = src.getLl(); + valueCount++; + size++; } // ==================== Array methods ==================== diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index cf73ee3..b4e61c6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; import org.junit.Assert; @@ -52,189 +54,214 @@ public static void setUpStatic() { AbstractLineSenderTest.setUpStatic(); } - // === Exact Type Match Tests === + // === BYTE coercion tests === @Test - public void testBoolean() throws Exception { - String table = "test_qwp_boolean"; + public void testByteToBooleanCoercionError() throws Exception { + String table = "test_qwp_byte_to_boolean_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("b", true) + .byteColumn("b", (byte) 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("b", false) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning BYTE and BOOLEAN but got: " + msg, + msg.contains("BYTE") && msg.contains("BOOLEAN") + ); } + } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "b\ttimestamp\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); + @Test + public void testByteToCharCoercionError() throws Exception { + String table = "test_qwp_byte_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .byteColumn("c", (byte) 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning BYTE and CHAR but got: " + msg, + msg.contains("BYTE") && msg.contains("CHAR") + ); + } } @Test - public void testByte() throws Exception { - String table = "test_qwp_byte"; + public void testByteToDate() throws Exception { + String table = "test_qwp_byte_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) -1) + .byteColumn("d", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("b", (short) 0) + .byteColumn("d", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testChar() throws Exception { - String table = "test_qwp_char"; + public void testByteToDecimal() throws Exception { + String table = "test_qwp_byte_to_decimal"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("c", 'A') + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", 'ü') // ü + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("c", '中') // 中 - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "c\ttimestamp\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "ü\t1970-01-01T00:00:02.000000000Z\n" + - "中\t1970-01-01T00:00:03.000000000Z\n", - "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal() throws Exception { - String table = "test_qwp_decimal"; + public void testByteToDecimal128() throws Exception { + String table = "test_qwp_byte_to_decimal128"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", "123.45") + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", "-999.99") + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", "0.01") - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(42_000, 3)) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDouble() throws Exception { - String table = "test_qwp_double"; + public void testByteToDecimal16() throws Exception { + String table = "test_qwp_byte_to_decimal16"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 42.5) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -1.0E10) + .byteColumn("d", (byte) -9) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MIN_VALUE) - .at(4_000_000, ChronoUnit.MICROS); - // NaN and Inf should be stored as null - sender.table(table) - .doubleColumn("d", Double.NaN) - .at(5_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.POSITIVE_INFINITY) - .at(6_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.NEGATIVE_INFINITY) - .at(7_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 7); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\ttimestamp\n" + - "42.5\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + - "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + - "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + - "null\t1970-01-01T00:00:05.000000000Z\n" + - "null\t1970-01-01T00:00:06.000000000Z\n" + - "null\t1970-01-01T00:00:07.000000000Z\n", - "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleArray() throws Exception { - String table = "test_qwp_double_array"; + public void testByteToDecimal256() throws Exception { + String table = "test_qwp_byte_to_decimal256"; useTable(table); - - double[] arr1d = createDoubleArray(5); - double[][] arr2d = createDoubleArray(2, 3); - double[][][] arr3d = createDoubleArray(1, 2, 3); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("a1", arr1d) - .doubleArray("a2", arr2d) - .doubleArray("a3", arr3d) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("d", (byte) -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimal() throws Exception { - String table = "test_qwp_double_to_decimal"; + public void testByteToDecimal64() throws Exception { + String table = "test_qwp_byte_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 123.45) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -42.10) + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -242,221 +269,244 @@ public void testDoubleToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_decimal_prec"; + public void testByteToDecimal8() throws Exception { + String table = "test_qwp_byte_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 123.456) + .byteColumn("d", (byte) 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("d", (byte) -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToByte() throws Exception { - String table = "test_qwp_double_to_byte"; + public void testByteToDouble() throws Exception { + String table = "test_qwp_byte_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 42.0) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("b", -100.0) + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToBytePrecisionLossError() throws Exception { - String table = "test_qwp_double_to_byte_prec"; + public void testByteToFloat() throws Exception { + String table = "test_qwp_byte_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 42.5) + .byteColumn("f", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("f", (byte) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("42.5") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToByteOverflowError() throws Exception { - String table = "test_qwp_double_to_byte_ovf"; + public void testByteToGeoHashCoercionError() throws Exception { + String table = "test_qwp_byte_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 200.0) + .byteColumn("g", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 200 out of range for BYTE") + "Expected coercion error mentioning BYTE but got: " + msg, + msg.contains("type coercion from BYTE to") && msg.contains("is not supported") ); } } @Test - public void testDoubleToFloat() throws Exception { - String table = "test_qwp_double_to_float"; + public void testByteToInt() throws Exception { + String table = "test_qwp_byte_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("f", 1.5) + .byteColumn("i", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("f", -42.25) + .byteColumn("i", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("i", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToInt() throws Exception { - String table = "test_qwp_double_to_int"; + public void testByteToLong() throws Exception { + String table = "test_qwp_byte_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 100_000.0) + .byteColumn("l", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("i", -42.0) + .byteColumn("l", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("l", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + - "100000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToIntPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_int_prec"; + public void testByteToLong256CoercionError() throws Exception { + String table = "test_qwp_byte_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 3.14) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("3.14") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to LONG256 is not supported") ); } } @Test - public void testDoubleToLong() throws Exception { - String table = "test_qwp_double_to_long"; + public void testByteToShort() throws Exception { + String table = "test_qwp_byte_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("l", 1_000_000.0) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("l", -42.0) + .byteColumn("s", Byte.MIN_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "1000000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToString() throws Exception { - String table = "test_qwp_double_to_string"; + public void testByteToString() throws Exception { + String table = "test_qwp_byte_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + "s STRING, " + @@ -466,385 +516,366 @@ public void testDoubleToString() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("s", 3.14) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("s", -42.0) + .byteColumn("s", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "s\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToSymbol() throws Exception { - String table = "test_qwp_double_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("sym", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "sym\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToVarchar() throws Exception { - String table = "test_qwp_double_to_varchar"; + public void testByteToSymbol() throws Exception { + String table = "test_qwp_byte_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("v", -42.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloat() throws Exception { - String table = "test_qwp_float"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("f", 1.5f) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", -42.25f) + .byteColumn("s", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", 0.0f) + .byteColumn("s", (byte) 0) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDecimal() throws Exception { - String table = "test_qwp_float_to_decimal"; + public void testByteToTimestamp() throws Exception { + String table = "test_qwp_byte_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .byteColumn("t", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .byteColumn("t", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1.50\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_decimal_prec"; + public void testByteToUuidCoercionError() throws Exception { + String table = "test_qwp_byte_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 1), " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.25f) + .byteColumn("u", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("scale=1") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to UUID is not supported") ); } } @Test - public void testFloatToDouble() throws Exception { - String table = "test_qwp_float_to_double"; + public void testByteToVarchar() throws Exception { + String table = "test_qwp_byte_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .byteColumn("v", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("v", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } + // === Exact Type Match Tests === + @Test - public void testFloatToInt() throws Exception { - String table = "test_qwp_float_to_int"; + public void testBoolean() throws Exception { + String table = "test_qwp_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 42.0f) + .boolColumn("b", true) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("i", -100.0f) + .boolColumn("b", false) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "b\ttimestamp\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testFloatToIntPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_int_prec"; + public void testByte() throws Exception { + String table = "test_qwp_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 3.14f) + .shortColumn("b", (short) -1) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") - ); } + + assertTableSizeEventually(table, 3); } @Test - public void testFloatToLong() throws Exception { - String table = "test_qwp_float_to_long"; + public void testChar() throws Exception { + String table = "test_qwp_char"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("l", 1000.0f) + .charColumn("c", 'A') .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", 'ü') // ü + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '中') // 中 + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "c\ttimestamp\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "ü\t1970-01-01T00:00:02.000000000Z\n" + + "中\t1970-01-01T00:00:03.000000000Z\n", + "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testFloatToString() throws Exception { - String table = "test_qwp_float_to_string"; + public void testDecimal() throws Exception { + String table = "test_qwp_decimal"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("s", 1.5f) + .decimalColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "0.01") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(42_000, 2)) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); - assertSqlEventually( - "s\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 4); } @Test - public void testFloatToSymbol() throws Exception { - String table = "test_qwp_float_to_symbol"; + public void testDouble() throws Exception { + String table = "test_qwp_double"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("sym", 1.5f) + .doubleColumn("d", 42.5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", -1.0E10) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MIN_VALUE) + .at(4_000_000, ChronoUnit.MICROS); + // NaN and Inf should be stored as null + sender.table(table) + .doubleColumn("d", Double.NaN) + .at(5_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.POSITIVE_INFINITY) + .at(6_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.NEGATIVE_INFINITY) + .at(7_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 7); assertSqlEventually( - "sym\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); + "d\ttimestamp\n" + + "42.5\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + + "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + + "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + + "null\t1970-01-01T00:00:05.000000000Z\n" + + "null\t1970-01-01T00:00:06.000000000Z\n" + + "null\t1970-01-01T00:00:07.000000000Z\n", + "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testFloatToVarchar() throws Exception { - String table = "test_qwp_float_to_varchar"; + public void testDoubleArray() throws Exception { + String table = "test_qwp_double_array"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + + double[] arr1d = createDoubleArray(5); + double[][] arr2d = createDoubleArray(2, 3); + double[][][] arr3d = createDoubleArray(1, 2, 3); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .doubleArray("a1", arr1d) + .doubleArray("a2", arr2d) + .doubleArray("a3", arr3d) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 1); - assertSqlEventually( - "v\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testInt() throws Exception { - String table = "test_qwp_int"; + public void testDoubleToDecimal() throws Exception { + String table = "test_qwp_double_to_decimal"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .intColumn("i", Integer.MIN_VALUE) + .doubleColumn("d", 123.45) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("i", 0) + .doubleColumn("d", -42.10) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", -42) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 2); assertSqlEventually( - "i\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n" + - "-42\t1970-01-01T00:00:04.000000000Z\n", - "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToBooleanCoercionError() throws Exception { - String table = "test_qwp_int_to_boolean_error"; + public void testDoubleToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 1) + .doubleColumn("d", 123.456) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and BOOLEAN but got: " + msg, - msg.contains("INT") && msg.contains("BOOLEAN") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") ); } } @Test - public void testIntToByte() throws Exception { - String table = "test_qwp_int_to_byte"; + public void testDoubleToByte() throws Exception { + String table = "test_qwp_double_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -854,29 +885,25 @@ public void testIntToByte() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 42) + .doubleColumn("b", 42.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("b", -128) + .doubleColumn("b", -100.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "b\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + "-100\t1970-01-01T00:00:02.000000000Z\n", "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToByteOverflowError() throws Exception { - String table = "test_qwp_int_to_byte_overflow"; + public void testDoubleToBytePrecisionLossError() throws Exception { + String table = "test_qwp_double_to_byte_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -886,661 +913,514 @@ public void testIntToByteOverflowError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 128) + .doubleColumn("b", 42.5) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("42.5") ); } } @Test - public void testIntToCharCoercionError() throws Exception { - String table = "test_qwp_int_to_char_error"; + public void testDoubleToByteOverflowError() throws Exception { + String table = "test_qwp_double_to_byte_ovf"; useTable(table); execute("CREATE TABLE " + table + " (" + - "c CHAR, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("c", 65) + .doubleColumn("b", 200.0) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and CHAR but got: " + msg, - msg.contains("INT") && msg.contains("CHAR") + "Expected overflow error but got: " + msg, + msg.contains("integer value 200 out of range for BYTE") ); } } @Test - public void testIntToDate() throws Exception { - String table = "test_qwp_int_to_date"; + public void testDoubleToFloat() throws Exception { + String table = "test_qwp_double_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 86_400_000 millis = 1 day sender.table(table) - .intColumn("d", 86_400_000) + .doubleColumn("f", 1.5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", 0) + .doubleColumn("f", -42.25) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal() throws Exception { - String table = "test_qwp_int_to_decimal"; + public void testDoubleToInt() throws Exception { + String table = "test_qwp_double_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("i", 100_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .doubleColumn("i", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "i\tts\n" + + "100000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal128() throws Exception { - String table = "test_qwp_int_to_decimal128"; + public void testDoubleToIntPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_int_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("i", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("3.14") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal16() throws Exception { - String table = "test_qwp_int_to_decimal16"; + public void testDoubleToLong() throws Exception { + String table = "test_qwp_double_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("l", 1_000_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .doubleColumn("l", -42.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal256() throws Exception { - String table = "test_qwp_int_to_decimal256"; + public void testDoubleToString() throws Exception { + String table = "test_qwp_double_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("s", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .doubleColumn("s", -42.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal64() throws Exception { - String table = "test_qwp_int_to_decimal64"; + public void testDoubleToSymbol() throws Exception { + String table = "test_qwp_double_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", Integer.MAX_VALUE) + .doubleColumn("sym", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( - "d\tts\n" + - "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "sym\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal8() throws Exception { - String table = "test_qwp_int_to_decimal8"; + public void testDoubleToVarchar() throws Exception { + String table = "test_qwp_double_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 5) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -9) + .doubleColumn("v", -42.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDouble() throws Exception { - String table = "test_qwp_int_to_double"; + public void testFloat() throws Exception { + String table = "test_qwp_float"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .floatColumn("f", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .floatColumn("f", -42.25f) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 3); } @Test - public void testIntToFloat() throws Exception { - String table = "test_qwp_int_to_float"; + public void testFloatToDecimal() throws Exception { + String table = "test_qwp_float_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("f", 42) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("f", -100) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("f", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1.50\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToGeoHashCoercionError() throws Exception { - String table = "test_qwp_int_to_geohash_error"; + public void testFloatToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + + "d DECIMAL(10, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("g", 42) + .floatColumn("d", 1.25f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning INT but got: " + msg, - msg.contains("type coercion from INT to") && msg.contains("is not supported") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("scale=1") ); } } @Test - public void testIntToLong() throws Exception { - String table = "test_qwp_int_to_long"; + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("l", 42) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("l", Integer.MAX_VALUE) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", -1) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "2147483647\t1970-01-01T00:00:02.000000000Z\n" + - "-1\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToLong256CoercionError() throws Exception { - String table = "test_qwp_int_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("v", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to LONG256 is not supported") - ); - } + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToShort() throws Exception { - String table = "test_qwp_int_to_short"; + public void testFloatToInt() throws Exception { + String table = "test_qwp_float_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 1000) + .floatColumn("i", 42.0f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("s", -32768) + .floatColumn("i", -100.0f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 32767) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToShortOverflowError() throws Exception { - String table = "test_qwp_int_to_short_overflow"; + public void testFloatToIntPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_int_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 32768) + .floatColumn("i", 3.14f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") ); } } @Test - public void testIntToString() throws Exception { - String table = "test_qwp_int_to_string"; + public void testFloatToLong() throws Exception { + String table = "test_qwp_float_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .floatColumn("l", 1000.0f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToSymbol() throws Exception { - String table = "test_qwp_int_to_symbol"; + public void testFloatToString() throws Exception { + String table = "test_qwp_float_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .floatColumn("s", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + "1.5\t1970-01-01T00:00:01.000000000Z\n", "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToTimestamp() throws Exception { - String table = "test_qwp_int_to_timestamp"; + public void testFloatToSymbol() throws Exception { + String table = "test_qwp_float_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 1_000_000 micros = 1 second sender.table(table) - .intColumn("t", 1_000_000) + .floatColumn("sym", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("t", 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); + "sym\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToUuidCoercionError() throws Exception { - String table = "test_qwp_int_to_uuid_error"; + public void testFloatToVarchar() throws Exception { + String table = "test_qwp_float_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("u", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to UUID is not supported") - ); - } - } - - @Test - public void testIntToVarchar() throws Exception { - String table = "test_qwp_int_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("v", 42) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "1.5\t1970-01-01T00:00:01.000000000Z\n", "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong() throws Exception { - String table = "test_qwp_long"; + public void testInt() throws Exception { + String table = "test_qwp_int"; useTable(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Long.MIN_VALUE is the null sentinel for LONG + // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .longColumn("l", Long.MIN_VALUE) + .intColumn("i", Integer.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("l", 0) + .intColumn("i", 0) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("l", Long.MAX_VALUE) + .intColumn("i", Integer.MAX_VALUE) .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", -42) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 4); assertSqlEventually( - "l\ttimestamp\n" + + "i\ttimestamp\n" + "null\t1970-01-01T00:00:01.000000000Z\n" + "0\t1970-01-01T00:00:02.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testLong256() throws Exception { - String table = "test_qwp_long256"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // All zeros - sender.table(table) - .long256Column("v", 0, 0, 0, 0) - .at(1_000_000, ChronoUnit.MICROS); - // Mixed values - sender.table(table) - .long256Column("v", 1, 2, 3, 4) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLongToBooleanCoercionError() throws Exception { - String table = "test_qwp_long_to_boolean_error"; + public void testIntToBooleanCoercionError() throws Exception { + String table = "test_qwp_int_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BOOLEAN, " + @@ -1550,22 +1430,22 @@ public void testLongToBooleanCoercionError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 1) + .intColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and BOOLEAN but got: " + msg, - msg.contains("LONG") && msg.contains("BOOLEAN") + "Expected error mentioning INT and BOOLEAN but got: " + msg, + msg.contains("INT") && msg.contains("BOOLEAN") ); } } @Test - public void testLongToByte() throws Exception { - String table = "test_qwp_long_to_byte"; + public void testIntToByte() throws Exception { + String table = "test_qwp_int_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -1575,13 +1455,13 @@ public void testLongToByte() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 42) + .intColumn("b", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("b", -128) + .intColumn("b", -128) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("b", 127) + .intColumn("b", 127) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1596,8 +1476,8 @@ public void testLongToByte() throws Exception { } @Test - public void testLongToByteOverflowError() throws Exception { - String table = "test_qwp_long_to_byte_overflow"; + public void testIntToByteOverflowError() throws Exception { + String table = "test_qwp_int_to_byte_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -1607,7 +1487,7 @@ public void testLongToByteOverflowError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 128) + .intColumn("b", 128) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1621,8 +1501,8 @@ public void testLongToByteOverflowError() throws Exception { } @Test - public void testLongToCharCoercionError() throws Exception { - String table = "test_qwp_long_to_char_error"; + public void testIntToCharCoercionError() throws Exception { + String table = "test_qwp_int_to_char_error"; useTable(table); execute("CREATE TABLE " + table + " (" + "c CHAR, " + @@ -1632,22 +1512,22 @@ public void testLongToCharCoercionError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("c", 65) + .intColumn("c", 65) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and CHAR but got: " + msg, - msg.contains("LONG") && msg.contains("CHAR") + "Expected error mentioning INT and CHAR but got: " + msg, + msg.contains("INT") && msg.contains("CHAR") ); } } @Test - public void testLongToDate() throws Exception { - String table = "test_qwp_long_to_date"; + public void testIntToDate() throws Exception { + String table = "test_qwp_int_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DATE, " + @@ -1656,11 +1536,12 @@ public void testLongToDate() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 86_400_000 millis = 1 day sender.table(table) - .longColumn("d", 86_400_000L) + .intColumn("d", 86_400_000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", 0L) + .intColumn("d", 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1674,21 +1555,21 @@ public void testLongToDate() throws Exception { } @Test - public void testLongToDecimal() throws Exception { - String table = "test_qwp_long_to_decimal"; + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DECIMAL(6, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1702,8 +1583,8 @@ public void testLongToDecimal() throws Exception { } @Test - public void testLongToDecimal128() throws Exception { - String table = "test_qwp_long_to_decimal128"; + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(38, 2), " + @@ -1713,25 +1594,29 @@ public void testLongToDecimal128() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 1_000_000_000L) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000L) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal16() throws Exception { - String table = "test_qwp_long_to_decimal16"; + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(4, 1), " + @@ -1741,25 +1626,29 @@ public void testLongToDecimal16() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal256() throws Exception { - String table = "test_qwp_long_to_decimal256"; + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(76, 2), " + @@ -1769,53 +1658,61 @@ public void testLongToDecimal256() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", Long.MAX_VALUE) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000_000L) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal32() throws Exception { - String table = "test_qwp_long_to_decimal32"; + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", Integer.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal8() throws Exception { - String table = "test_qwp_long_to_decimal8"; + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(2, 1), " + @@ -1825,25 +1722,29 @@ public void testLongToDecimal8() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 5) + .intColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -9) + .intColumn("d", -9) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDouble() throws Exception { - String table = "test_qwp_long_to_double"; + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DOUBLE, " + @@ -1853,10 +1754,10 @@ public void testLongToDouble() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1870,8 +1771,8 @@ public void testLongToDouble() throws Exception { } @Test - public void testLongToFloat() throws Exception { - String table = "test_qwp_long_to_float"; + public void testIntToFloat() throws Exception { + String table = "test_qwp_int_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + "f FLOAT, " + @@ -1881,25 +1782,29 @@ public void testLongToFloat() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("f", 42) + .intColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("f", -100) + .intColumn("f", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "f\tts\n" + "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long_to_geohash_error"; + public void testIntToGeoHashCoercionError() throws Exception { + String table = "test_qwp_int_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + "g GEOHASH(4c), " + @@ -1909,61 +1814,121 @@ public void testLongToGeoHashCoercionError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("g", 42) + .intColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning LONG but got: " + msg, - msg.contains("type coercion from LONG to") && msg.contains("is not supported") + "Expected coercion error mentioning INT but got: " + msg, + msg.contains("type coercion from INT to") && msg.contains("is not supported") ); } } @Test - public void testLongToInt() throws Exception { - String table = "test_qwp_long_to_int"; + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in INT range should succeed sender.table(table) - .longColumn("i", 42) + .intColumn("l", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("i", -1) + .intColumn("l", Integer.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + + "l\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToIntOverflowError() throws Exception { - String table = "test_qwp_long_to_int_overflow"; + public void testIntToLong256CoercionError() throws Exception { + String table = "test_qwp_int_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("i", (long) Integer.MAX_VALUE + 1) + .intColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to LONG256 is not supported") + ); + } + } + + @Test + public void testIntToShort() throws Exception { + String table = "test_qwp_int_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 1000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -32768) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 32767) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToShortOverflowError() throws Exception { + String table = "test_qwp_int_to_short_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1971,24 +1936,117 @@ public void testLongToIntOverflowError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected overflow error but got: " + msg, - msg.contains("integer value 2147483648 out of range for INT") + msg.contains("integer value 32768 out of range for SHORT") ); } } @Test - public void testLongToLong256CoercionError() throws Exception { - String table = "test_qwp_long_to_long256_error"; + public void testIntToString() throws Exception { + String table = "test_qwp_int_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v LONG256, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .intColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToSymbol() throws Exception { + String table = "test_qwp_int_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToTimestamp() throws Exception { + String table = "test_qwp_int_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // 1_000_000 micros = 1 second + sender.table(table) + .intColumn("t", 1_000_000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("t", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToUuidCoercionError() throws Exception { + String table = "test_qwp_int_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1996,255 +2054,1130 @@ public void testLongToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to LONG256 is not supported") + msg.contains("type coercion from INT to UUID is not supported") ); } } @Test - public void testLongToShort() throws Exception { - String table = "test_qwp_long_to_short"; + public void testIntToVarchar() throws Exception { + String table = "test_qwp_int_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in SHORT range should succeed sender.table(table) - .longColumn("s", 42) + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", -1) + .intColumn("v", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("v", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong() throws Exception { + String table = "test_qwp_long"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG + sender.table(table) + .longColumn("l", Long.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testLong256() throws Exception { + String table = "test_qwp_long256"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros + sender.table(table) + .long256Column("v", 0, 0, 0, 0) + .at(1_000_000, ChronoUnit.MICROS); + // Mixed values + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToBooleanCoercionError() throws Exception { + String table = "test_qwp_long_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 1) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and BOOLEAN but got: " + msg, + msg.contains("LONG") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testLongToByte() throws Exception { + String table = "test_qwp_long_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToByteOverflowError() throws Exception { + String table = "test_qwp_long_to_byte_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 128) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); + } + } + + @Test + public void testLongToCharCoercionError() throws Exception { + String table = "test_qwp_long_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("c", 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and CHAR but got: " + msg, + msg.contains("LONG") && msg.contains("CHAR") + ); + } + } + + @Test + public void testLongToDate() throws Exception { + String table = "test_qwp_long_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 86_400_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", 0L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 1_000_000_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", Long.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDouble() throws Exception { + String table = "test_qwp_long_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToFloat() throws Exception { + String table = "test_qwp_long_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("f", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("f", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("g", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning LONG but got: " + msg, + msg.contains("type coercion from LONG to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in INT range should succeed + sender.table(table) + .longColumn("i", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("i", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToIntOverflowError() throws Exception { + String table = "test_qwp_long_to_int_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("i", (long) Integer.MAX_VALUE + 1) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 2147483648 out of range for INT") + ); + } + } + + @Test + public void testLongToLong256CoercionError() throws Exception { + String table = "test_qwp_long_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to LONG256 is not supported") + ); + } + } + + @Test + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToShortOverflowError() throws Exception { + String table = "test_qwp_long_to_short_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("s", 32768) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") + ); + } + } + + @Test + public void testLongToString() throws Exception { + String table = "test_qwp_long_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToSymbol() throws Exception { + String table = "test_qwp_long_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToTimestamp() throws Exception { + String table = "test_qwp_long_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("t", 1_000_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("t", 0L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToUuidCoercionError() throws Exception { + String table = "test_qwp_long_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("u", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to UUID is not supported") + ); + } + } + + @Test + public void testLongToVarchar() throws Exception { + String table = "test_qwp_long_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("v", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testMultipleRowsAndBatching() throws Exception { + String table = "test_qwp_multiple_rows"; + useTable(table); + + int rowCount = 1000; + try (QwpWebSocketSender sender = createQwpSender()) { + for (int i = 0; i < rowCount; i++) { + sender.table(table) + .symbol("sym", "s" + (i % 10)) + .longColumn("val", i) + .doubleColumn("dbl", i * 1.5) + .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); + } + sender.flush(); + } + + assertTableSizeEventually(table, rowCount); + } + + @Test + public void testShort() throws Exception { + String table = "test_qwp_short"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Short.MIN_VALUE is the null sentinel for SHORT + sender.table(table) + .shortColumn("s", Short.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testShortToDecimal128() throws Exception { + String table = "test_qwp_short_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", Short.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", Short.MIN_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "32767.00\t1970-01-01T00:00:01.000000000Z\n" + + "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal16() throws Exception { + String table = "test_qwp_short_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal256() throws Exception { + String table = "test_qwp_short_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal32() throws Exception { + String table = "test_qwp_short_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToShortOverflowError() throws Exception { - String table = "test_qwp_long_to_short_overflow"; + public void testShortToDecimal64() throws Exception { + String table = "test_qwp_short_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 32768) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToString() throws Exception { - String table = "test_qwp_long_to_string"; + public void testShortToDecimal8() throws Exception { + String table = "test_qwp_short_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .shortColumn("d", (short) 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", Long.MAX_VALUE) + .shortColumn("d", (short) -9) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToSymbol() throws Exception { - String table = "test_qwp_long_to_symbol"; + public void testShortToInt() throws Exception { + String table = "test_qwp_short_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .shortColumn("i", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", -1) + .shortColumn("i", Short.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + + "i\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToTimestamp() throws Exception { - String table = "test_qwp_long_to_timestamp"; + public void testShortToLong() throws Exception { + String table = "test_qwp_short_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("t", 1_000_000L) + .shortColumn("l", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("t", 0L) + .shortColumn("l", Short.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToUuidCoercionError() throws Exception { - String table = "test_qwp_long_to_uuid_error"; + public void testShortToBooleanCoercionError() throws Exception { + String table = "test_qwp_short_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("u", 42) + .shortColumn("b", (short) 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to UUID is not supported") + "Expected error mentioning SHORT and BOOLEAN but got: " + msg, + msg.contains("SHORT") && msg.contains("BOOLEAN") ); } } @Test - public void testLongToVarchar() throws Exception { - String table = "test_qwp_long_to_varchar"; + public void testShortToByte() throws Exception { + String table = "test_qwp_short_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .shortColumn("b", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("v", Long.MAX_VALUE) + .shortColumn("b", (short) -128) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "v\tts\n" + + "b\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testMultipleRowsAndBatching() throws Exception { - String table = "test_qwp_multiple_rows"; + public void testShortToByteOverflowError() throws Exception { + String table = "test_qwp_short_to_byte_overflow"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); - int rowCount = 1000; try (QwpWebSocketSender sender = createQwpSender()) { - for (int i = 0; i < rowCount; i++) { - sender.table(table) - .symbol("sym", "s" + (i % 10)) - .longColumn("val", i) - .doubleColumn("dbl", i * 1.5) - .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); - } + sender.table(table) + .shortColumn("b", (short) 128) + .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); } - - assertTableSizeEventually(table, rowCount); } @Test - public void testShort() throws Exception { - String table = "test_qwp_short"; + public void testShortToCharCoercionError() throws Exception { + String table = "test_qwp_short_to_char_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Short.MIN_VALUE is the null sentinel for SHORT sender.table(table) - .shortColumn("s", Short.MIN_VALUE) + .shortColumn("c", (short) 65) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning SHORT and CHAR but got: " + msg, + msg.contains("SHORT") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 3); } @Test - public void testShortToDecimal128() throws Exception { - String table = "test_qwp_short_to_decimal128"; + public void testShortToDate() throws Exception { + String table = "test_qwp_short_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 1000 millis = 1 second sender.table(table) - .shortColumn("d", Short.MAX_VALUE) + .shortColumn("d", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", Short.MIN_VALUE) + .shortColumn("d", (short) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -2252,17 +3185,17 @@ public void testShortToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "32767.00\t1970-01-01T00:00:01.000000000Z\n" + - "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal16() throws Exception { - String table = "test_qwp_short_to_decimal16"; + public void testShortToDouble() throws Exception { + String table = "test_qwp_short_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); @@ -2286,171 +3219,230 @@ public void testShortToDecimal16() throws Exception { } @Test - public void testShortToDecimal256() throws Exception { - String table = "test_qwp_short_to_decimal256"; + public void testShortToFloat() throws Exception { + String table = "test_qwp_short_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .shortColumn("f", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .shortColumn("f", (short) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal32() throws Exception { - String table = "test_qwp_short_to_decimal32"; + public void testShortToGeoHashCoercionError() throws Exception { + String table = "test_qwp_short_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .shortColumn("g", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning SHORT but got: " + msg, + msg.contains("type coercion from SHORT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testShortToLong256CoercionError() throws Exception { + String table = "test_qwp_short_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from SHORT to LONG256 is not supported") + ); + } + } + + @Test + public void testShortToString() throws Exception { + String table = "test_qwp_short_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) -100) + .shortColumn("s", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal64() throws Exception { - String table = "test_qwp_short_to_decimal64"; + public void testShortToSymbol() throws Exception { + String table = "test_qwp_short_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .shortColumn("s", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .shortColumn("s", (short) -1) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal8() throws Exception { - String table = "test_qwp_short_to_decimal8"; + public void testShortToTimestamp() throws Exception { + String table = "test_qwp_short_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 5) + .shortColumn("t", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -9) + .shortColumn("t", (short) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToInt() throws Exception { - String table = "test_qwp_short_to_int"; + public void testShortToUuidCoercionError() throws Exception { + String table = "test_qwp_short_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("i", (short) 42) + .shortColumn("u", (short) 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("i", Short.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from SHORT to UUID is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToLong() throws Exception { - String table = "test_qwp_short_to_long"; + public void testShortToVarchar() throws Exception { + String table = "test_qwp_short_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("l", (short) 42) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("l", Short.MAX_VALUE) + .shortColumn("v", (short) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("v", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + + "v\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test @@ -2759,6 +3751,261 @@ public void testWriteAllTypesInOneRow() throws Exception { assertTableSizeEventually(table, 1); } + // === Decimal cross-width coercion tests === + + @Test + public void testDecimal256ToDecimal64() throws Exception { + String table = "test_qwp_dec256_to_dec64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL256 wire type to DECIMAL64 column + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal256ToDecimal128() throws Exception { + String table = "test_qwp_dec256_to_dec128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal64ToDecimal128() throws Exception { + String table = "test_qwp_dec64_to_dec128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL64 wire type to DECIMAL128 column (widening) + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal64ToDecimal256() throws Exception { + String table = "test_qwp_dec64_to_dec256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal128ToDecimal64() throws Exception { + String table = "test_qwp_dec128_to_dec64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal128ToDecimal256() throws Exception { + String table = "test_qwp_dec128_to_dec256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimalRescale() throws Exception { + String table = "test_qwp_decimal_rescale"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 4), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Send scale=2 wire data to scale=4 column: server should rescale + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-100, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.4500\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal256ToDecimal64OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec64_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Create a value that fits in Decimal256 but overflows Decimal64 + // Decimal256 with hi bits set will overflow 64-bit storage + Decimal256 bigValue = Decimal256.fromBigDecimal(new java.math.BigDecimal("99999999999999999999.99")); + sender.table(table) + .decimalColumn("d", bigValue) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); + } + } + + @Test + public void testDecimal256ToDecimal8OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec8_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // 999.9 with scale=1 → unscaled 9999, which doesn't fit in a byte (-128..127) + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(9999, 1)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); + } + } + // === Helper Methods === private QwpWebSocketSender createQwpSender() { From dd4bb6232e99a0c14c69a99b823acd1025fce6a7 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 16 Feb 2026 00:07:18 +0000 Subject: [PATCH 011/230] wip 5 --- .../cutlass/qwp/client/QwpSenderTest.java | 4032 ++++++++++++++++- 1 file changed, 4030 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index b4e61c6..6b7e8d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -3609,13 +3609,13 @@ public void testTimestampMicrosToNanos() throws Exception { String table = "test_qwp_timestamp_micros_to_nanos"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP WITH TIME ZONE, " + + "ts_col TIMESTAMP_NS, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + long tsMicros = 1_645_747_200_111_111L; // 2022-02-25T00:00:00Z sender.table(table) .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); @@ -3623,6 +3623,11 @@ public void testTimestampMicrosToNanos() throws Exception { } assertTableSizeEventually(table, 1); + // Microseconds scaled to nanoseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test @@ -4006,6 +4011,4029 @@ public void testDecimal256ToDecimal8OverflowError() throws Exception { } } + @Test + public void testStringToBoolean() throws Exception { + String table = "test_qwp_string_to_boolean"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "true") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "false") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "1") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "0") + .at(4_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "TRUE") + .at(5_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 5); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n" + + "true\t1970-01-01T00:00:03.000000000Z\n" + + "false\t1970-01-01T00:00:04.000000000Z\n" + + "true\t1970-01-01T00:00:05.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToBooleanParseError() throws Exception { + String table = "test_qwp_string_to_boolean_err"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "yes") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse boolean from string") + ); + } + } + + @Test + public void testStringToByte() throws Exception { + String table = "test_qwp_string_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "-128") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "127") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToByteParseError() throws Exception { + String table = "test_qwp_string_to_byte_err"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "abc") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse BYTE from string") + ); + } + } + + @Test + public void testStringToDate() throws Exception { + String table = "test_qwp_string_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "2022-02-25T00:00:00.000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "d\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal64() throws Exception { + String table = "test_qwp_string_to_dec64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal128() throws Exception { + String table = "test_qwp_string_to_dec128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal256() throws Exception { + String table = "test_qwp_string_to_dec256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDouble() throws Exception { + String table = "test_qwp_string_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-2.718") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.718\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToFloat() throws Exception { + String table = "test_qwp_string_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("f", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", "-2.5") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.5\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToGeoHash() throws Exception { + String table = "test_qwp_string_to_geohash"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(5c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("g", "s24se") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("g", "u33dc") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "g\tts\n" + + "s24se\t1970-01-01T00:00:01.000000000Z\n" + + "u33dc\t1970-01-01T00:00:02.000000000Z\n", + "SELECT g, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToInt() throws Exception { + String table = "test_qwp_string_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("i", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "-100") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "0") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToLong() throws Exception { + String table = "test_qwp_string_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("l", "1000000000000") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("l", "-1") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "1000000000000\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToLong256() throws Exception { + String table = "test_qwp_string_to_long256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("l", "0x01") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "l\tts\n" + + "0x01\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToShort() throws Exception { + String table = "test_qwp_string_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "1000") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "-32768") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "32767") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToTimestamp() throws Exception { + String table = "test_qwp_string_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testBoolToString() throws Exception { + String table = "test_qwp_bool_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("s", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .boolColumn("s", false) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testBoolToVarchar() throws Exception { + String table = "test_qwp_bool_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .boolColumn("v", false) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimalToString() throws Exception { + String table = "test_qwp_decimal_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("s", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("s", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimalToVarchar() throws Exception { + String table = "test_qwp_decimal_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testSymbolToString() throws Exception { + String table = "test_qwp_symbol_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testSymbolToVarchar() throws Exception { + String table = "test_qwp_symbol_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testTimestampToString() throws Exception { + String table = "test_qwp_timestamp_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + sender.table(table) + .timestampColumn("s", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testTimestampToVarchar() throws Exception { + String table = "test_qwp_timestamp_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + sender.table(table) + .timestampColumn("v", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testCharToString() throws Exception { + String table = "test_qwp_char_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("s", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("s", 'Z') + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testCharToVarchar() throws Exception { + String table = "test_qwp_char_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("v", 'Z') + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToShort() throws Exception { + String table = "test_qwp_double_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 100.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("v", -200.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "100\t1970-01-01T00:00:01.000000000Z\n" + + "-200\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToByte() throws Exception { + String table = "test_qwp_float_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 7.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("v", -100.0f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "7\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToShort() throws Exception { + String table = "test_qwp_float_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 42.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("v", -1000.0f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong256ToString() throws Exception { + String table = "test_qwp_long256_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("s", 1, 2, 3, 4) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); + } + + @Test + public void testLong256ToVarchar() throws Exception { + String table = "test_qwp_long256_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); + } + + @Test + public void testStringToDecimal8() throws Exception { + String table = "test_qwp_string_to_dec8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "1.5") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-9.9") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-9.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal16() throws Exception { + String table = "test_qwp_string_to_dec16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "12.5") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.9") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "12.5\t1970-01-01T00:00:01.000000000Z\n" + + "-99.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal32() throws Exception { + String table = "test_qwp_string_to_dec32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "1234.56") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1234.56\t1970-01-01T00:00:01.000000000Z\n" + + "-999.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToTimestampNs() throws Exception { + String table = "test_qwp_string_to_timestamp_ns"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP_NS, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("ts_col", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); + } + + @Test + public void testUuidToString() throws Exception { + String table = "test_qwp_uuid_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); + } + + @Test + public void testUuidToVarchar() throws Exception { + String table = "test_qwp_uuid_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); + } + + // === SYMBOL negative coercion tests === + + @Test + public void testSymbolToBooleanCoercionError() throws Exception { + String table = "test_qwp_symbol_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testSymbolToByteCoercionError() throws Exception { + String table = "test_qwp_symbol_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("BYTE") + ); + } + } + + @Test + public void testSymbolToCharCoercionError() throws Exception { + String table = "test_qwp_symbol_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("CHAR") + ); + } + } + + @Test + public void testSymbolToDateCoercionError() throws Exception { + String table = "test_qwp_symbol_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("DATE") + ); + } + } + + @Test + public void testSymbolToDecimalCoercionError() throws Exception { + String table = "test_qwp_symbol_to_decimal_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("DECIMAL") + ); + } + } + + @Test + public void testSymbolToDoubleCoercionError() throws Exception { + String table = "test_qwp_symbol_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testSymbolToFloatCoercionError() throws Exception { + String table = "test_qwp_symbol_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testSymbolToGeoHashCoercionError() throws Exception { + String table = "test_qwp_symbol_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testSymbolToIntCoercionError() throws Exception { + String table = "test_qwp_symbol_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("INT") + ); + } + } + + @Test + public void testSymbolToLongCoercionError() throws Exception { + String table = "test_qwp_symbol_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("LONG") + ); + } + } + + @Test + public void testSymbolToLong256CoercionError() throws Exception { + String table = "test_qwp_symbol_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("LONG256") + ); + } + } + + @Test + public void testSymbolToShortCoercionError() throws Exception { + String table = "test_qwp_symbol_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("SHORT") + ); + } + } + + @Test + public void testSymbolToTimestampCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testSymbolToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testSymbolToUuidCoercionError() throws Exception { + String table = "test_qwp_symbol_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("UUID") + ); + } + } + + // === Null coercion tests === + + @Test + public void testNullStringToBoolean() throws Exception { + String table = "test_qwp_null_string_to_boolean"; + useTable(table); + execute("CREATE TABLE " + table + " (b BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "true") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToChar() throws Exception { + String table = "test_qwp_null_string_to_char"; + useTable(table); + execute("CREATE TABLE " + table + " (c CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("c", "A") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("c", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "c\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT c, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToDate() throws Exception { + String table = "test_qwp_null_string_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (d DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "2022-02-25T00:00:00.000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToDecimal() throws Exception { + String table = "test_qwp_null_string_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (d DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToGeoHash() throws Exception { + String table = "test_qwp_null_string_to_geohash"; + useTable(table); + execute("CREATE TABLE " + table + " (g GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("g", "s09wh") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("g", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "g\tts\n" + + "s09wh\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT g, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToLong256() throws Exception { + String table = "test_qwp_null_string_to_long256"; + useTable(table); + execute("CREATE TABLE " + table + " (l LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("l", "0x01") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("l", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "0x01\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToNumeric() throws Exception { + String table = "test_qwp_null_string_to_numeric"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "l LONG, " + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("i", "42") + .stringColumn("l", "100") + .stringColumn("d", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", null) + .stringColumn("l", null) + .stringColumn("d", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tl\td\tts\n" + + "42\t100\t3.14\t1970-01-01T00:00:01.000000000Z\n" + + "null\tnull\tnull\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, l, d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToSymbol() throws Exception { + String table = "test_qwp_null_string_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "alpha") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToTimestamp() throws Exception { + String table = "test_qwp_null_string_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (t TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("t", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToTimestampNs() throws Exception { + String table = "test_qwp_null_string_to_timestamp_ns"; + useTable(table); + execute("CREATE TABLE " + table + " (t TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("t", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToUuid() throws Exception { + String table = "test_qwp_null_string_to_uuid"; + useTable(table); + execute("CREATE TABLE " + table + " (u UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("u", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "u\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT u, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullSymbolToString() throws Exception { + String table = "test_qwp_null_symbol_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (s STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullSymbolToVarchar() throws Exception { + String table = "test_qwp_null_symbol_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + // === BOOLEAN negative tests === + + @Test + public void testBooleanToByteCoercionError() throws Exception { + String table = "test_qwp_boolean_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("BYTE") + ); + } + } + + @Test + public void testBooleanToShortCoercionError() throws Exception { + String table = "test_qwp_boolean_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("SHORT") + ); + } + } + + @Test + public void testBooleanToIntCoercionError() throws Exception { + String table = "test_qwp_boolean_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("INT") + ); + } + } + + @Test + public void testBooleanToLongCoercionError() throws Exception { + String table = "test_qwp_boolean_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG") + ); + } + } + + @Test + public void testBooleanToFloatCoercionError() throws Exception { + String table = "test_qwp_boolean_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testBooleanToDoubleCoercionError() throws Exception { + String table = "test_qwp_boolean_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testBooleanToDateCoercionError() throws Exception { + String table = "test_qwp_boolean_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DATE") + ); + } + } + + @Test + public void testBooleanToUuidCoercionError() throws Exception { + String table = "test_qwp_boolean_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("UUID") + ); + } + } + + @Test + public void testBooleanToLong256CoercionError() throws Exception { + String table = "test_qwp_boolean_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG256") + ); + } + } + + @Test + public void testBooleanToGeoHashCoercionError() throws Exception { + String table = "test_qwp_boolean_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testBooleanToTimestampCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testBooleanToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testBooleanToCharCoercionError() throws Exception { + String table = "test_qwp_boolean_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("CHAR") + ); + } + } + + @Test + public void testBooleanToSymbolCoercionError() throws Exception { + String table = "test_qwp_boolean_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testBooleanToDecimalCoercionError() throws Exception { + String table = "test_qwp_boolean_to_decimal_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DECIMAL") + ); + } + } + + // === FLOAT negative tests === + + @Test + public void testFloatToBooleanCoercionError() throws Exception { + String table = "test_qwp_float_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testFloatToCharCoercionError() throws Exception { + String table = "test_qwp_float_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("CHAR") + ); + } + } + + @Test + public void testFloatToDateCoercionError() throws Exception { + String table = "test_qwp_float_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToGeoHashCoercionError() throws Exception { + String table = "test_qwp_float_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToUuidCoercionError() throws Exception { + String table = "test_qwp_float_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToLong256CoercionError() throws Exception { + String table = "test_qwp_float_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + // === DOUBLE negative tests === + + @Test + public void testDoubleToBooleanCoercionError() throws Exception { + String table = "test_qwp_double_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testDoubleToCharCoercionError() throws Exception { + String table = "test_qwp_double_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("CHAR") + ); + } + } + + @Test + public void testDoubleToDateCoercionError() throws Exception { + String table = "test_qwp_double_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testDoubleToGeoHashCoercionError() throws Exception { + String table = "test_qwp_double_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testDoubleToUuidCoercionError() throws Exception { + String table = "test_qwp_double_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testDoubleToLong256CoercionError() throws Exception { + String table = "test_qwp_double_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + // ==================== CHAR negative tests ==================== + + @Test + public void testCharToBooleanCoercionError() throws Exception { + String table = "test_qwp_char_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testCharToSymbolCoercionError() throws Exception { + String table = "test_qwp_char_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testCharToByteCoercionError() throws Exception { + String table = "test_qwp_char_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("BYTE") + ); + } + } + + @Test + public void testCharToShortCoercionError() throws Exception { + String table = "test_qwp_char_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("SHORT") + ); + } + } + + @Test + public void testCharToIntCoercionError() throws Exception { + String table = "test_qwp_char_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("INT") + ); + } + } + + @Test + public void testCharToLongCoercionError() throws Exception { + String table = "test_qwp_char_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG") + ); + } + } + + @Test + public void testCharToFloatCoercionError() throws Exception { + String table = "test_qwp_char_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testCharToDoubleCoercionError() throws Exception { + String table = "test_qwp_char_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testCharToDateCoercionError() throws Exception { + String table = "test_qwp_char_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DATE") + ); + } + } + + @Test + public void testCharToUuidCoercionError() throws Exception { + String table = "test_qwp_char_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("UUID") + ); + } + } + + @Test + public void testCharToLong256CoercionError() throws Exception { + String table = "test_qwp_char_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG256") + ); + } + } + + @Test + public void testCharToGeoHashCoercionError() throws Exception { + String table = "test_qwp_char_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("GEOHASH") + ); + } + } + + // ==================== LONG256 negative tests ==================== + + @Test + public void testLong256ToBooleanCoercionError() throws Exception { + String table = "test_qwp_long256_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testLong256ToCharCoercionError() throws Exception { + String table = "test_qwp_long256_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("CHAR") + ); + } + } + + @Test + public void testLong256ToSymbolCoercionError() throws Exception { + String table = "test_qwp_long256_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testLong256ToByteCoercionError() throws Exception { + String table = "test_qwp_long256_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToShortCoercionError() throws Exception { + String table = "test_qwp_long256_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToIntCoercionError() throws Exception { + String table = "test_qwp_long256_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToLongCoercionError() throws Exception { + String table = "test_qwp_long256_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToFloatCoercionError() throws Exception { + String table = "test_qwp_long256_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToDoubleCoercionError() throws Exception { + String table = "test_qwp_long256_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToDateCoercionError() throws Exception { + String table = "test_qwp_long256_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToUuidCoercionError() throws Exception { + String table = "test_qwp_long256_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long256_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + // ==================== UUID negative tests ==================== + + @Test + public void testUuidToBooleanCoercionError() throws Exception { + String table = "test_qwp_uuid_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testUuidToCharCoercionError() throws Exception { + String table = "test_qwp_uuid_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("CHAR") + ); + } + } + + @Test + public void testUuidToSymbolCoercionError() throws Exception { + String table = "test_qwp_uuid_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testUuidToByteCoercionError() throws Exception { + String table = "test_qwp_uuid_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToIntCoercionError() throws Exception { + String table = "test_qwp_uuid_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToLongCoercionError() throws Exception { + String table = "test_qwp_uuid_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToFloatCoercionError() throws Exception { + String table = "test_qwp_uuid_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToDoubleCoercionError() throws Exception { + String table = "test_qwp_uuid_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToDateCoercionError() throws Exception { + String table = "test_qwp_uuid_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToLong256CoercionError() throws Exception { + String table = "test_qwp_uuid_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToGeoHashCoercionError() throws Exception { + String table = "test_qwp_uuid_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + // === TIMESTAMP negative coercion tests === + + @Test + public void testTimestampToBooleanCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testTimestampToByteCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("BYTE") + ); + } + } + + @Test + public void testTimestampToShortCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("SHORT") + ); + } + } + + @Test + public void testTimestampToIntCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("INT") + ); + } + } + + @Test + public void testTimestampToLongCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG") + ); + } + } + + @Test + public void testTimestampToFloatCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testTimestampToDoubleCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testTimestampToDateCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("DATE") + ); + } + } + + @Test + public void testTimestampToUuidCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("UUID") + ); + } + } + + @Test + public void testTimestampToLong256CoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG256") + ); + } + } + + @Test + public void testTimestampToGeoHashCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testTimestampToCharCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("CHAR") + ); + } + } + + @Test + public void testTimestampToSymbolCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testTimestampToDecimalCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_decimal_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("DECIMAL") + ); + } + } + + // === DECIMAL negative coercion tests === + + @Test + public void testDecimalToBooleanCoercionError() throws Exception { + String table = "test_qwp_decimal_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testDecimalToByteCoercionError() throws Exception { + String table = "test_qwp_decimal_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("BYTE") + ); + } + } + + @Test + public void testDecimalToShortCoercionError() throws Exception { + String table = "test_qwp_decimal_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SHORT") + ); + } + } + + @Test + public void testDecimalToIntCoercionError() throws Exception { + String table = "test_qwp_decimal_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("INT") + ); + } + } + + @Test + public void testDecimalToLongCoercionError() throws Exception { + String table = "test_qwp_decimal_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG") + ); + } + } + + @Test + public void testDecimalToFloatCoercionError() throws Exception { + String table = "test_qwp_decimal_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testDecimalToDoubleCoercionError() throws Exception { + String table = "test_qwp_decimal_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testDecimalToDateCoercionError() throws Exception { + String table = "test_qwp_decimal_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DATE") + ); + } + } + + @Test + public void testDecimalToUuidCoercionError() throws Exception { + String table = "test_qwp_decimal_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("UUID") + ); + } + } + + @Test + public void testDecimalToLong256CoercionError() throws Exception { + String table = "test_qwp_decimal_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG256") + ); + } + } + + @Test + public void testDecimalToGeoHashCoercionError() throws Exception { + String table = "test_qwp_decimal_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testDecimalToTimestampCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testDecimalToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testDecimalToCharCoercionError() throws Exception { + String table = "test_qwp_decimal_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("CHAR") + ); + } + } + + @Test + public void testDecimalToSymbolCoercionError() throws Exception { + String table = "test_qwp_decimal_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SYMBOL") + ); + } + } + + // === DOUBLE_ARRAY negative coercion tests === + + @Test + public void testDoubleArrayToIntCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("INT") + ); + } + } + + @Test + public void testDoubleArrayToStringCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_string_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("STRING") + ); + } + } + + @Test + public void testDoubleArrayToSymbolCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testDoubleArrayToTimestampCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("TIMESTAMP") + ); + } + } + + // ==================== Additional null coercion tests ==================== + + @Test + public void testNullStringToVarchar() throws Exception { + String table = "test_qwp_null_string_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullSymbolToSymbol() throws Exception { + String table = "test_qwp_null_symbol_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "alpha") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToByte() throws Exception { + String table = "test_qwp_null_string_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (b BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToShort() throws Exception { + String table = "test_qwp_null_string_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (s SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToFloat() throws Exception { + String table = "test_qwp_null_string_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (f FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("f", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + // ==================== Additional positive coercion test ==================== + + @Test + public void testStringToVarchar() throws Exception { + String table = "test_qwp_string_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + // ==================== Additional parse error tests ==================== + + @Test + public void testStringToIntParseError() throws Exception { + String table = "test_qwp_string_to_int_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse INT from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToLongParseError() throws Exception { + String table = "test_qwp_string_to_long_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse LONG from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToShortParseError() throws Exception { + String table = "test_qwp_string_to_short_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse SHORT from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToFloatParseError() throws Exception { + String table = "test_qwp_string_to_float_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse FLOAT from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToDoubleParseError() throws Exception { + String table = "test_qwp_string_to_double_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse DOUBLE from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToDateParseError() throws Exception { + String table = "test_qwp_string_to_date_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_date") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse DATE from string") && msg.contains("not_a_date") + ); + } + } + + @Test + public void testStringToTimestampParseError() throws Exception { + String table = "test_qwp_string_to_timestamp_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_timestamp") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse timestamp from string") && msg.contains("not_a_timestamp") + ); + } + } + + @Test + public void testStringToUuidParseError() throws Exception { + String table = "test_qwp_string_to_uuid_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not-a-uuid") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse UUID from string") && msg.contains("not-a-uuid") + ); + } + } + + @Test + public void testStringToLong256ParseError() throws Exception { + String table = "test_qwp_string_to_long256_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_long256") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse long256 from string") && msg.contains("not_a_long256") + ); + } + } + + @Test + public void testStringToGeoHashParseError() throws Exception { + String table = "test_qwp_string_to_geohash_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "!!!") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse geohash from string") && msg.contains("!!!") + ); + } + } + // === Helper Methods === private QwpWebSocketSender createQwpSender() { From f609b5a795fc85f5cba107a39a8a5c0c541b5221 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 22 Feb 2026 00:28:59 +0000 Subject: [PATCH 012/230] wip 9 --- .../questdb/client/test/cutlass/qwp/client/QwpSenderTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index 6b7e8d9..02106f0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -868,7 +868,7 @@ public void testDoubleToDecimalPrecisionLossError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") + msg.contains("cannot be converted to") && msg.contains("123.456") && msg.contains("scale=2") ); } } @@ -1204,7 +1204,7 @@ public void testFloatToDecimalPrecisionLossError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("scale=1") + msg.contains("cannot be converted to") && msg.contains("scale=1") ); } } From eb9531ddf9e85623e79856c987ee32f19832ebb7 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 22 Feb 2026 02:56:03 +0000 Subject: [PATCH 013/230] wip 11 --- .../qwp/client/QwpWebSocketEncoder.java | 446 ++--- .../qwp/client/QwpWebSocketSender.java | 11 + .../qwp/protocol/OffHeapAppendMemory.java | 161 ++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 1606 +++++++---------- .../qwp/protocol/OffHeapAppendMemoryTest.java | 266 +++ 5 files changed, 1249 insertions(+), 1241 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index c71e158..ef473de 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -28,37 +28,26 @@ import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * Encodes ILP v4 messages for WebSocket transport. *

- * This encoder can write to either an internal {@link NativeBufferWriter} (default) - * or an external {@link QwpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}. + * This encoder reads column data from off-heap {@link io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory} + * buffers in {@link QwpTableBuffer.ColumnBuffer} and uses bulk {@code putBlockOfBytes} for fixed-width + * types where wire format matches native byte order. *

- * When using an external buffer, the encoder writes directly to it without intermediate copies, - * enabling zero-copy WebSocket frame construction. + * Types that use bulk copy (native byte-order on wire): + * BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DATE, UUID, LONG256 *

- * Usage with external buffer (zero-copy): - * 

- * WebSocketSendBuffer buf = client.getSendBuffer();
- * buf.beginBinaryFrame();
- * encoder.setBuffer(buf);
- * encoder.encode(tableData, false);
- * FrameInfo frame = buf.endBinaryFrame();
- * client.sendFrame(frame);
- *
+ * Types that require element-by-element encoding: + * BOOLEAN (bit-packed on wire), TIMESTAMP (Gorilla), DECIMAL64/128/256 (big-endian on wire) */ public class QwpWebSocketEncoder implements QuietCloseable { - /** - * Encoding flag for Gorilla-encoded timestamps. - */ public static final byte ENCODING_GORILLA = 0x01; - /** - * Encoding flag for uncompressed timestamps. - */ public static final byte ENCODING_UNCOMPRESSED = 0x00; private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); private QwpBufferWriter buffer; @@ -85,43 +74,16 @@ public void close() { } } - /** - * Encodes a complete ILP v4 message from a table buffer. - * - * @param tableBuffer the table buffer containing row data - * @param useSchemaRef whether to use schema reference mode - * @return the number of bytes written - */ public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { buffer.reset(); - - // Write message header with placeholder for payload length writeHeader(1, 0); int payloadStart = buffer.getPosition(); - - // Encode table data encodeTable(tableBuffer, useSchemaRef); - - // Patch payload length int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); - return buffer.getPosition(); } - /** - * Encodes a complete ILP v4 message with delta symbol dictionary encoding. - *

- * This method sends only new symbols (delta) since the last confirmed watermark, - * and uses global symbol IDs instead of per-column local indices. - * - * @param tableBuffer the table buffer containing row data - * @param globalDict the global symbol dictionary - * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) - * @param batchMaxId the highest symbol ID used in this batch - * @param useSchemaRef whether to use schema reference mode - * @return the number of bytes written - */ public int encodeWithDeltaDict( QwpTableBuffer tableBuffer, GlobalSymbolDictionary globalDict, @@ -130,101 +92,51 @@ public int encodeWithDeltaDict( boolean useSchemaRef ) { buffer.reset(); - - // Calculate delta range int deltaStart = confirmedMaxId + 1; int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); - - // Set delta dictionary flag byte savedFlags = flags; flags |= FLAG_DELTA_SYMBOL_DICT; - - // Write message header with placeholder for payload length writeHeader(1, 0); int payloadStart = buffer.getPosition(); - - // Write symbol delta section (before tables) buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } - - // Encode table data (symbol columns will use global IDs) encodeTableWithGlobalSymbols(tableBuffer, useSchemaRef); - - // Patch payload length int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); - - // Restore flags flags = savedFlags; - return buffer.getPosition(); } - /** - * Returns the underlying buffer. - *

- * If an external buffer was set via {@link #setBuffer(QwpBufferWriter)}, - * that buffer is returned. Otherwise, returns the internal buffer. - */ public QwpBufferWriter getBuffer() { return buffer; } - /** - * Returns true if delta symbol dictionary encoding is enabled. - */ public boolean isDeltaSymbolDictEnabled() { return (flags & FLAG_DELTA_SYMBOL_DICT) != 0; } - /** - * Returns true if Gorilla encoding is enabled. - */ public boolean isGorillaEnabled() { return (flags & FLAG_GORILLA) != 0; } - /** - * Returns true if currently using an external buffer. - */ public boolean isUsingExternalBuffer() { return buffer != ownedBuffer; } - /** - * Resets the encoder for a new message. - *

- * If using an external buffer, this only resets the internal state (flags). - * The external buffer's reset is the caller's responsibility. - * If using the internal buffer, resets both the buffer and internal state. - */ public void reset() { if (!isUsingExternalBuffer()) { buffer.reset(); } } - /** - * Sets an external buffer for encoding. - *

- * When set, the encoder writes directly to this buffer instead of its internal buffer. - * The caller is responsible for managing the external buffer's lifecycle. - *

- * Pass {@code null} to revert to using the internal buffer. - * - * @param externalBuffer the external buffer to use, or null to use internal buffer - */ public void setBuffer(QwpBufferWriter externalBuffer) { this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; } - /** - * Sets the delta symbol dictionary flag. - */ public void setDeltaSymbolDictEnabled(boolean enabled) { if (enabled) { flags |= FLAG_DELTA_SYMBOL_DICT; @@ -233,9 +145,6 @@ public void setDeltaSymbolDictEnabled(boolean enabled) { } } - /** - * Sets whether Gorilla timestamp encoding is enabled. - */ public void setGorillaEnabled(boolean enabled) { if (enabled) { flags |= FLAG_GORILLA; @@ -244,73 +153,54 @@ public void setGorillaEnabled(boolean enabled) { } } - /** - * Writes the ILP v4 message header. - * - * @param tableCount number of tables in the message - * @param payloadLength payload length (can be 0 if patched later) - */ public void writeHeader(int tableCount, int payloadLength) { - // Magic "ILP4" buffer.putByte((byte) 'I'); buffer.putByte((byte) 'L'); buffer.putByte((byte) 'P'); buffer.putByte((byte) '4'); - - // Version buffer.putByte(VERSION_1); - - // Flags buffer.putByte(flags); - - // Table count (uint16, little-endian) buffer.putShort((short) tableCount); - - // Payload length (uint32, little-endian) buffer.putInt(payloadLength); } - /** - * Encodes a single column. - */ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); + long dataAddr = col.getDataAddress(); - // Write null bitmap if column is nullable if (colDef.isNullable()) { - writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + writeNullBitmap(col, rowCount); } - // Write column data based on type switch (col.getType()) { case TYPE_BOOLEAN: - writeBooleanColumn(col.getBooleanValues(), valueCount); + writeBooleanColumn(dataAddr, valueCount); break; case TYPE_BYTE: - writeByteColumn(col.getByteValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, valueCount); break; case TYPE_SHORT: case TYPE_CHAR: - writeShortColumn(col.getShortValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); break; case TYPE_INT: - writeIntColumn(col.getIntValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_LONG: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_FLOAT: - writeFloatColumn(col.getFloatValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_DOUBLE: - writeDoubleColumn(col.getDoubleValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: - writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + writeTimestampColumn(dataAddr, valueCount, useGorilla); break; case TYPE_DATE: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_STRING: case TYPE_VARCHAR: @@ -320,10 +210,12 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, writeSymbolColumn(col, valueCount); break; case TYPE_UUID: - writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + // Stored as lo+hi contiguously, matching wire order + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); break; case TYPE_LONG256: - writeLong256Column(col.getLong256Values(), valueCount); + // Stored as 4 contiguous longs per value + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); break; case TYPE_DOUBLE_ARRAY: writeDoubleArrayColumn(col, valueCount); @@ -332,77 +224,70 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, writeLongArrayColumn(col, valueCount); break; case TYPE_DECIMAL64: - writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL128: - writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL256: - writeDecimal256Column(col.getDecimalScale(), - col.getDecimal256Hh(), col.getDecimal256Hl(), - col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); break; default: throw new IllegalStateException("Unknown column type: " + col.getType()); } } - /** - * Encodes a single column using global symbol IDs for SYMBOL type. - * All other column types are encoded the same as encodeColumn. - */ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); - // Write null bitmap if column is nullable if (colDef.isNullable()) { - writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + writeNullBitmap(col, rowCount); } - // For symbol columns, use global IDs; for all others, use standard encoding if (col.getType() == TYPE_SYMBOL) { writeSymbolColumnWithGlobalIds(col, valueCount); } else { - // Write column data based on type (same as encodeColumn) + // Delegate to standard encoding for all other types + long dataAddr = col.getDataAddress(); switch (col.getType()) { case TYPE_BOOLEAN: - writeBooleanColumn(col.getBooleanValues(), valueCount); + writeBooleanColumn(dataAddr, valueCount); break; case TYPE_BYTE: - writeByteColumn(col.getByteValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, valueCount); break; case TYPE_SHORT: case TYPE_CHAR: - writeShortColumn(col.getShortValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); break; case TYPE_INT: - writeIntColumn(col.getIntValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_LONG: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_FLOAT: - writeFloatColumn(col.getFloatValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_DOUBLE: - writeDoubleColumn(col.getDoubleValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: - writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + writeTimestampColumn(dataAddr, valueCount, useGorilla); break; case TYPE_DATE: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_STRING: case TYPE_VARCHAR: writeStringColumn(col.getStringValues(), valueCount); break; case TYPE_UUID: - writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); break; case TYPE_LONG256: - writeLong256Column(col.getLong256Values(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); break; case TYPE_DOUBLE_ARRAY: writeDoubleArrayColumn(col, valueCount); @@ -411,15 +296,13 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC writeLongArrayColumn(col, valueCount); break; case TYPE_DECIMAL64: - writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL128: - writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL256: - writeDecimal256Column(col.getDecimalScale(), - col.getDecimal256Hh(), col.getDecimal256Hl(), - col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); break; default: throw new IllegalStateException("Unknown column type: " + col.getType()); @@ -427,9 +310,6 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC } } - /** - * Encodes a single table from the buffer. - */ private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); @@ -445,7 +325,6 @@ private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); } - // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); @@ -454,10 +333,6 @@ private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { } } - /** - * Encodes a single table from the buffer using global symbol IDs. - * This is used with delta dictionary encoding. - */ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean useSchemaRef) { QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); @@ -473,7 +348,6 @@ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean us writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); } - // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); @@ -483,16 +357,16 @@ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean us } /** - * Writes boolean column data (bit-packed). + * Writes boolean column data (bit-packed on wire). + * Reads individual bytes from off-heap and packs into bits. */ - private void writeBooleanColumn(boolean[] values, int count) { + private void writeBooleanColumn(long addr, int count) { int packedSize = (count + 7) / 8; - for (int i = 0; i < packedSize; i++) { byte b = 0; for (int bit = 0; bit < 8; bit++) { int idx = i * 8 + bit; - if (idx < count && values[idx]) { + if (idx < count && Unsafe.getUnsafe().getByte(addr + idx) != 0) { b |= (1 << bit); } } @@ -500,34 +374,44 @@ private void writeBooleanColumn(boolean[] values, int count) { } } - private void writeByteColumn(byte[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putByte(values[i]); - } - } - - private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { + /** + * Writes Decimal128 values in big-endian wire format. + * Reads hi/lo pairs from off-heap (stored as hi, lo per value). + */ + private void writeDecimal128Column(byte scale, long addr, int count) { buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLongBE(high[i]); - buffer.putLongBE(low[i]); + long offset = (long) i * 16; + long hi = Unsafe.getUnsafe().getLong(addr + offset); + long lo = Unsafe.getUnsafe().getLong(addr + offset + 8); + buffer.putLongBE(hi); + buffer.putLongBE(lo); } } - private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { + /** + * Writes Decimal256 values in big-endian wire format. + * Reads hh/hl/lh/ll quads from off-heap (stored contiguously per value). + */ + private void writeDecimal256Column(byte scale, long addr, int count) { buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLongBE(hh[i]); - buffer.putLongBE(hl[i]); - buffer.putLongBE(lh[i]); - buffer.putLongBE(ll[i]); + long offset = (long) i * 32; + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 8)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 16)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 24)); } } - private void writeDecimal64Column(byte scale, long[] values, int count) { + /** + * Writes Decimal64 values in big-endian wire format. + * Reads longs from off-heap. + */ + private void writeDecimal64Column(byte scale, long addr, int count) { buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLongBE(values[i]); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + (long) i * 8)); } } @@ -555,32 +439,6 @@ private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) } } - private void writeDoubleColumn(double[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putDouble(values[i]); - } - } - - private void writeFloatColumn(float[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putFloat(values[i]); - } - } - - private void writeIntColumn(int[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putInt(values[i]); - } - } - - private void writeLong256Column(long[] values, int count) { - // Flat array: 4 longs per value, little-endian (least significant first) - // values layout: [long0, long1, long2, long3] per row - for (int i = 0; i < count * 4; i++) { - buffer.putLong(values[i]); - } - } - private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); @@ -605,37 +463,26 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { } } - private void writeLongColumn(long[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putLong(values[i]); - } - } - /** - * Writes a null bitmap from bit-packed long array. + * Writes a null bitmap from off-heap memory. + * On little-endian platforms, the byte layout of the long-packed bitmap + * in memory matches the wire format, enabling bulk copy. */ - private void writeNullBitmapPacked(long[] nullsPacked, int count) { - int bitmapSize = (count + 7) / 8; - - for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { - int longIndex = byteIdx >>> 3; - int byteInLong = byteIdx & 7; - byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); - buffer.putByte(b); - } - } - - private void writeShortColumn(short[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putShort(values[i]); + private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) { + long nullAddr = col.getNullBitmapAddress(); + if (nullAddr != 0) { + int bitmapSize = (rowCount + 7) / 8; + buffer.putBlockOfBytes(nullAddr, bitmapSize); + } else { + // Non-nullable column shouldn't reach here, but write zeros as fallback + int bitmapSize = (rowCount + 7) / 8; + for (int i = 0; i < bitmapSize; i++) { + buffer.putByte((byte) 0); + } } } - /** - * Writes a string column with offset array. - */ private void writeStringColumn(String[] strings, int count) { - // Calculate total data length int totalDataLen = 0; for (int i = 0; i < count; i++) { if (strings[i] != null) { @@ -643,7 +490,6 @@ private void writeStringColumn(String[] strings, int count) { } } - // Write offset array int runningOffset = 0; buffer.putInt(0); for (int i = 0; i < count; i++) { @@ -653,7 +499,6 @@ private void writeStringColumn(String[] strings, int count) { buffer.putInt(runningOffset); } - // Write string data for (int i = 0; i < count; i++) { if (strings[i] != null) { buffer.putUtf8(strings[i]); @@ -663,133 +508,98 @@ private void writeStringColumn(String[] strings, int count) { /** * Writes a symbol column with dictionary. - * Format: - * - Dictionary length (varint) - * - Dictionary entries (length-prefixed UTF-8 strings) - * - Symbol indices (varints, one per value) + * Reads local symbol indices from off-heap data buffer. */ private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) { - // Get symbol data from column buffer - int[] symbolIndices = col.getSymbolIndices(); + long dataAddr = col.getDataAddress(); String[] dictionary = col.getSymbolDictionary(); - // Write dictionary buffer.putVarint(dictionary.length); for (String symbol : dictionary) { buffer.putString(symbol); } - // Write symbol indices (one per non-null value) for (int i = 0; i < count; i++) { - buffer.putVarint(symbolIndices[i]); + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); } } /** * Writes a symbol column using global IDs (for delta dictionary mode). - * Format: - * - Global symbol IDs (varints, one per value) - *

- * The dictionary is not included here because it's written at the message level - * in delta format. + * Reads from auxiliary data buffer if available, otherwise falls back to local indices. */ private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) { - int[] globalIds = col.getGlobalSymbolIds(); - if (globalIds == null) { - // Fall back to local indices if no global IDs stored - int[] symbolIndices = col.getSymbolIndices(); + long auxAddr = col.getAuxDataAddress(); + if (auxAddr == 0) { + // Fall back to local indices + long dataAddr = col.getDataAddress(); for (int i = 0; i < count; i++) { - buffer.putVarint(symbolIndices[i]); + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); } } else { - // Write global symbol IDs for (int i = 0; i < count; i++) { - buffer.putVarint(globalIds[i]); + int globalId = Unsafe.getUnsafe().getInt(auxAddr + (long) i * 4); + buffer.putVarint(globalId); } } } - /** - * Writes a table header with full schema. - */ private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) { - // Table name buffer.putString(tableName); - - // Row count (varint) buffer.putVarint(rowCount); - - // Column count (varint) buffer.putVarint(columns.length); - - // Schema mode: full schema (0x00) buffer.putByte(SCHEMA_MODE_FULL); - - // Column definitions (name + type for each) for (QwpColumnDef col : columns) { buffer.putString(col.getName()); buffer.putByte(col.getWireTypeCode()); } } - /** - * Writes a table header with schema reference. - */ private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { - // Table name buffer.putString(tableName); - - // Row count (varint) buffer.putVarint(rowCount); - - // Column count (varint) buffer.putVarint(columnCount); - - // Schema mode: reference (0x01) buffer.putByte(SCHEMA_MODE_REFERENCE); - - // Schema hash (8 bytes) buffer.putLong(schemaHash); } /** * Writes a timestamp column with optional Gorilla compression. - *

- * When Gorilla encoding is enabled and applicable (3+ timestamps with - * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. - * Otherwise, falls back to uncompressed encoding. + * Reads longs from off-heap. For Gorilla encoding, creates a temporary + * on-heap array since the Gorilla encoder requires long[]. */ - private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { - if (useGorilla && count > 2 && QwpGorillaEncoder.canUseGorilla(values, count)) { - // Write Gorilla encoding flag - buffer.putByte(ENCODING_GORILLA); - - // Calculate size needed and ensure buffer has capacity - int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); - buffer.ensureCapacity(encodedSize); - - // Encode timestamps to buffer - int bytesWritten = gorillaEncoder.encodeTimestamps( - buffer.getBufferPtr() + buffer.getPosition(), - buffer.getCapacity() - buffer.getPosition(), - values, - count - ); - buffer.skip(bytesWritten); + private void writeTimestampColumn(long addr, int count, boolean useGorilla) { + if (useGorilla && count > 2) { + // Extract to temp array for Gorilla encoder (which requires long[]) + long[] values = new long[count]; + for (int i = 0; i < count; i++) { + values[i] = Unsafe.getUnsafe().getLong(addr + (long) i * 8); + } + + if (QwpGorillaEncoder.canUseGorilla(values, count)) { + buffer.putByte(ENCODING_GORILLA); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); + buffer.ensureCapacity(encodedSize); + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + values, + count + ); + buffer.skip(bytesWritten); + } else { + buffer.putByte(ENCODING_UNCOMPRESSED); + // Bulk copy for uncompressed path + buffer.putBlockOfBytes(addr, (long) count * 8); + } } else { - // Write uncompressed if (useGorilla) { buffer.putByte(ENCODING_UNCOMPRESSED); } - writeLongColumn(values, count); - } - } - - private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { - // Little-endian: lo first, then hi - for (int i = 0; i < count; i++) { - buffer.putLong(lowBits[i]); - buffer.putLong(highBits[i]); + // Bulk copy for uncompressed timestamps + buffer.putBlockOfBytes(addr, (long) count * 8); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d428dbb..dabda70 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1420,6 +1420,17 @@ public void close() { client = null; } encoder.close(); + // Close all table buffers to free off-heap column memory + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + tb.close(); + } + } + } tableBuffers.clear(); LOG.info("QwpWebSocketSender closed"); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java new file mode 100644 index 0000000..f4c14cc --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -0,0 +1,161 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +/** + * Lightweight append-only off-heap buffer for columnar data storage. + *

+ * This buffer provides typed append operations (putByte, putShort, etc.) backed by + * native memory allocated via {@link Unsafe}. Memory is tracked under + * {@link MemoryTag#NATIVE_ILP_RSS} for precise accounting. + *

+ * Growth strategy: capacity doubles on each resize via {@link Unsafe#realloc}. + */ +public class OffHeapAppendMemory implements QuietCloseable { + + private static final int DEFAULT_INITIAL_CAPACITY = 128; + + private long pageAddress; + private long appendAddress; + private long capacity; + + public OffHeapAppendMemory() { + this(DEFAULT_INITIAL_CAPACITY); + } + + public OffHeapAppendMemory(long initialCapacity) { + this.capacity = Math.max(initialCapacity, 8); + this.pageAddress = Unsafe.malloc(this.capacity, MemoryTag.NATIVE_ILP_RSS); + this.appendAddress = pageAddress; + } + + /** + * Returns the append offset (number of bytes written). + */ + public long getAppendOffset() { + return appendAddress - pageAddress; + } + + /** + * Returns the base address of the buffer. + */ + public long pageAddress() { + return pageAddress; + } + + /** + * Returns the address at the given byte offset from the start. + */ + public long addressOf(long offset) { + return pageAddress + offset; + } + + /** + * Resets the append position to 0 without freeing memory. + */ + public void truncate() { + appendAddress = pageAddress; + } + + /** + * Sets the append position to the given byte offset. + * Used for truncateTo operations on column buffers. + */ + public void jumpTo(long offset) { + appendAddress = pageAddress + offset; + } + + public void putByte(byte value) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(appendAddress, value); + appendAddress++; + } + + public void putBoolean(boolean value) { + putByte(value ? (byte) 1 : (byte) 0); + } + + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(appendAddress, value); + appendAddress += 2; + } + + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(appendAddress, value); + appendAddress += 4; + } + + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(appendAddress, value); + appendAddress += 8; + } + + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(appendAddress, value); + appendAddress += 4; + } + + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(appendAddress, value); + appendAddress += 8; + } + + /** + * Advances the append position by the given number of bytes without writing. + */ + public void skip(long bytes) { + ensureCapacity(bytes); + appendAddress += bytes; + } + + @Override + public void close() { + if (pageAddress != 0) { + Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS); + pageAddress = 0; + appendAddress = 0; + capacity = 0; + } + } + + private void ensureCapacity(long needed) { + long used = appendAddress - pageAddress; + if (used + needed > capacity) { + long newCapacity = Math.max(capacity * 2, used + needed); + pageAddress = Unsafe.realloc(pageAddress, capacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); + capacity = newCapacity; + appendAddress = pageAddress + used; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index f70dd98..9e97b4c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,8 +33,11 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Decimals; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import java.util.Arrays; @@ -43,10 +46,11 @@ /** * Buffers rows for a single table in columnar format. *

- * This buffer accumulates row data column by column, allowing efficient - * encoding to the ILP v4 wire format. + * Fixed-width column data is stored off-heap via {@link OffHeapAppendMemory} for zero-GC + * buffering and bulk copy to network buffers. Variable-width data (strings, symbol + * dictionaries, arrays) remains on-heap. */ -public class QwpTableBuffer { +public class QwpTableBuffer implements QuietCloseable { private final String tableName; private final ObjList columns; @@ -70,24 +74,43 @@ public QwpTableBuffer(String tableName) { } /** - * Returns the table name. + * Cancels the current in-progress row. + *

+ * This removes any column values added since the last {@link #nextRow()} call. + * If no values have been added for the current row, this is a no-op. */ - public String getTableName() { - return tableName; + public void cancelCurrentRow() { + // Reset sequential access cursor + columnAccessCursor = 0; + // Truncate each column back to the committed row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + col.truncateTo(rowCount); + } } /** - * Returns the number of rows buffered. + * Clears the buffer completely, including column definitions. + * Frees all off-heap memory. */ - public int getRowCount() { - return rowCount; + public void clear() { + for (int i = 0, n = columns.size(); i < n; i++) { + columns.get(i).close(); + } + columns.clear(); + columnNameToIndex.clear(); + fastColumns = null; + columnAccessCursor = 0; + rowCount = 0; + schemaHash = 0; + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; } - /** - * Returns the number of columns. - */ - public int getColumnCount() { - return columns.size(); + @Override + public void close() { + clear(); } /** @@ -97,6 +120,13 @@ public ColumnBuffer getColumn(int index) { return columns.get(index); } + /** + * Returns the number of columns. + */ + public int getColumnCount() { + return columns.size(); + } + /** * Returns the column definitions (cached for efficiency). */ @@ -167,38 +197,10 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) } /** - * Advances to the next row. - *

- * This should be called after all column values for the current row have been set. - */ - public void nextRow() { - // Reset sequential access cursor for the next row - columnAccessCursor = 0; - // Ensure all columns have the same row count - for (int i = 0, n = columns.size(); i < n; i++) { - ColumnBuffer col = fastColumns[i]; - // If column wasn't set for this row, add a null - while (col.size < rowCount + 1) { - col.addNull(); - } - } - rowCount++; - } - - /** - * Cancels the current in-progress row. - *

- * This removes any column values added since the last {@link #nextRow()} call. - * If no values have been added for the current row, this is a no-op. + * Returns the number of rows buffered. */ - public void cancelCurrentRow() { - // Reset sequential access cursor - columnAccessCursor = 0; - // Truncate each column back to the committed row count - for (int i = 0, n = columns.size(); i < n; i++) { - ColumnBuffer col = fastColumns[i]; - col.truncateTo(rowCount); - } + public int getRowCount() { + return rowCount; } /** @@ -218,7 +220,33 @@ public long getSchemaHash() { } /** - * Resets the buffer for reuse. + * Returns the table name. + */ + public String getTableName() { + return tableName; + } + + /** + * Advances to the next row. + *

+ * This should be called after all column values for the current row have been set. + */ + public void nextRow() { + // Reset sequential access cursor for the next row + columnAccessCursor = 0; + // Ensure all columns have the same row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + // If column wasn't set for this row, add a null + while (col.size < rowCount + 1) { + col.addNull(); + } + } + rowCount++; + } + + /** + * Resets the buffer for reuse. Keeps column definitions and allocated memory. */ public void reset() { for (int i = 0, n = columns.size(); i < n; i++) { @@ -229,639 +257,228 @@ public void reset() { } /** - * Clears the buffer completely, including column definitions. + * Returns the element size in bytes for a fixed-width column type. + * Returns 0 for variable-width types (string, arrays). */ - public void clear() { - columns.clear(); - columnNameToIndex.clear(); - fastColumns = null; - columnAccessCursor = 0; - rowCount = 0; - schemaHash = 0; - schemaHashComputed = false; - columnDefsCacheValid = false; - cachedColumnDefs = null; + static int elementSize(byte type) { + switch (type) { + case TYPE_BOOLEAN: + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_SYMBOL: + case TYPE_FLOAT: + return 4; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + case TYPE_DECIMAL64: + case TYPE_DOUBLE: + return 8; + case TYPE_UUID: + case TYPE_DECIMAL128: + return 16; + case TYPE_LONG256: + case TYPE_DECIMAL256: + return 32; + default: + return 0; + } } /** * Column buffer for a single column. + *

+ * Fixed-width data is stored off-heap in {@link OffHeapAppendMemory} for zero-GC + * operation and efficient bulk copy to network buffers. */ - public static class ColumnBuffer { + public static class ColumnBuffer implements QuietCloseable { final String name; final byte type; final boolean nullable; + final int elemSize; private int size; // Total row count (including nulls) private int valueCount; // Actual stored values (excludes nulls) - private int capacity; - - // Storage for different types - private boolean[] booleanValues; - private byte[] byteValues; - private short[] shortValues; - private int[] intValues; - private long[] longValues; - private float[] floatValues; - private double[] doubleValues; - private String[] stringValues; - private long[] uuidHigh; - private long[] uuidLow; - // Long256 stored as flat array: 4 longs per value (avoids inner array allocation) - private long[] long256Values; - // Array storage (double/long arrays - variable length per row) - // Each row stores: [nDims (1B)][dim1..dimN (4B each)][flattened data] - // We track per-row metadata separately from the actual data - private byte[] arrayDims; // nDims per row - private int[] arrayShapes; // Flattened shape data (all dimensions concatenated) - private int arrayShapeOffset; // Current write offset in arrayShapes - private double[] doubleArrayData; // Flattened double values - private long[] longArrayData; // Flattened long values - private int arrayDataOffset; // Current write offset in data arrays - private int arrayRowCapacity; // Capacity for array row count - - // Null tracking - bit-packed for memory efficiency (1 bit per row vs 8 bits with boolean[]) - private long[] nullBitmapPacked; + // Off-heap data buffer for fixed-width types + private OffHeapAppendMemory dataBuffer; + + // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only) + private OffHeapAppendMemory auxBuffer; + + // Off-heap null bitmap (bit-packed, 1 bit per row) + private long nullBufPtr; + private int nullBufCapRows; private boolean hasNulls; - // Symbol specific + // On-heap capacity for variable-width arrays (string values, array dims) + private int onHeapCapacity; + + // On-heap storage for variable-width types + private String[] stringValues; + + // Array storage (double/long arrays - variable length per row) + private byte[] arrayDims; + private int[] arrayShapes; + private int arrayShapeOffset; + private double[] doubleArrayData; + private long[] longArrayData; + private int arrayDataOffset; + + // Symbol specific (dictionary stays on-heap) private CharSequenceIntHashMap symbolDict; private ObjList symbolList; - private int[] symbolIndices; - - // Global symbol IDs for delta encoding (parallel to symbolIndices) - private int[] globalSymbolIds; private int maxGlobalSymbolId = -1; // Decimal storage - // All values in a decimal column must share the same scale - // For Decimal64: single long per value (64-bit unscaled) - // For Decimal128: two longs per value (128-bit unscaled: high, low) - // For Decimal256: four longs per value (256-bit unscaled: hh, hl, lh, ll) - private byte decimalScale = -1; // Shared scale for column (-1 = not set) - private final Decimal256 rescaleTemp = new Decimal256(); // Reusable temp for rescaling - private long[] decimal64Values; // Decimal64: one long per value - private long[] decimal128High; // Decimal128: high 64 bits - private long[] decimal128Low; // Decimal128: low 64 bits - private long[] decimal256Hh; // Decimal256: bits 255-192 - private long[] decimal256Hl; // Decimal256: bits 191-128 - private long[] decimal256Lh; // Decimal256: bits 127-64 - private long[] decimal256Ll; // Decimal256: bits 63-0 + private byte decimalScale = -1; + private final Decimal256 rescaleTemp = new Decimal256(); public ColumnBuffer(String name, byte type, boolean nullable) { this.name = name; this.type = type; this.nullable = nullable; + this.elemSize = elementSize(type); this.size = 0; this.valueCount = 0; - this.capacity = 16; this.hasNulls = false; + this.onHeapCapacity = 16; - allocateStorage(type, capacity); + allocateStorage(type); if (nullable) { - // Bit-packed: 64 bits per long, so we need (capacity + 63) / 64 longs - nullBitmapPacked = new long[(capacity + 63) >>> 6]; + nullBufCapRows = 64; // multiple of 64 + long sizeBytes = (long) nullBufCapRows >>> 3; + nullBufPtr = Unsafe.calloc(sizeBytes, MemoryTag.NATIVE_ILP_RSS); } } - public String getName() { - return name; - } - - public byte getType() { - return type; - } - - public int getSize() { - return size; - } - - /** - * Returns the number of actual stored values (excludes nulls). - */ - public int getValueCount() { - return valueCount; - } - - public boolean hasNulls() { - return hasNulls; + public void addBoolean(boolean value) { + dataBuffer.putByte(value ? (byte) 1 : (byte) 0); + valueCount++; + size++; } - /** - * Returns the bit-packed null bitmap. - * Each long contains 64 bits, bit 0 of long 0 = row 0, bit 1 of long 0 = row 1, etc. - */ - public long[] getNullBitmapPacked() { - return nullBitmapPacked; + public void addByte(byte value) { + dataBuffer.putByte(value); + valueCount++; + size++; } - /** - * Returns the null bitmap as boolean array (for backward compatibility). - * This creates a new array, so prefer getNullBitmapPacked() for efficiency. - */ - public boolean[] getNullBitmap() { - if (nullBitmapPacked == null) { - return null; + public void addDecimal128(Decimal128 value) { + if (value == null || value.isNull()) { + addNull(); + return; } - boolean[] result = new boolean[size]; - for (int i = 0; i < size; i++) { - result[i] = isNull(i); + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getHigh(), value.getLow()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + valueCount++; + size++; + return; } - return result; + dataBuffer.putLong(value.getHigh()); + dataBuffer.putLong(value.getLow()); + valueCount++; + size++; } - /** - * Checks if the row at the given index is null. - */ - public boolean isNull(int index) { - if (nullBitmapPacked == null) { - return false; + public void addDecimal256(Decimal256 value) { + if (value == null || value.isNull()) { + addNull(); + return; } - int longIndex = index >>> 6; - int bitIndex = index & 63; - return (nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0; - } - - public boolean[] getBooleanValues() { - return booleanValues; - } - - public byte[] getByteValues() { - return byteValues; - } - - public short[] getShortValues() { - return shortValues; - } - - public int[] getIntValues() { - return intValues; + Decimal256 src = value; + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.copyFrom(value); + rescaleTemp.rescale(decimalScale); + src = rescaleTemp; + } + dataBuffer.putLong(src.getHh()); + dataBuffer.putLong(src.getHl()); + dataBuffer.putLong(src.getLh()); + dataBuffer.putLong(src.getLl()); + valueCount++; + size++; } - public long[] getLongValues() { - return longValues; + public void addDecimal64(Decimal64 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + dataBuffer.putLong(value.getValue()); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getValue()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + dataBuffer.putLong(rescaleTemp.getLl()); + } else { + dataBuffer.putLong(value.getValue()); + } + valueCount++; + size++; } - public float[] getFloatValues() { - return floatValues; + public void addDouble(double value) { + dataBuffer.putDouble(value); + valueCount++; + size++; } - public double[] getDoubleValues() { - return doubleValues; + public void addDoubleArray(double[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (double v : values) { + doubleArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; } - public String[] getStringValues() { - return stringValues; + public void addDoubleArray(double[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + ensureArrayCapacity(2, dim0 * dim1); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (double[] row : values) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; } - public long[] getUuidHigh() { - return uuidHigh; - } - - public long[] getUuidLow() { - return uuidLow; - } - - /** - * Returns Long256 values as flat array (4 longs per value). - * Use getLong256Value(index, component) for indexed access. - */ - public long[] getLong256Values() { - return long256Values; - } - - /** - * Returns a component of a Long256 value. - * @param index value index - * @param component component 0-3 - */ - public long getLong256Value(int index, int component) { - return long256Values[index * 4 + component]; - } - - // ==================== Decimal getters ==================== - - /** - * Returns the shared scale for this decimal column. - * Returns -1 if no values have been added yet. - */ - public byte getDecimalScale() { - return decimalScale; - } - - /** - * Returns the Decimal64 values (one long per value). - */ - public long[] getDecimal64Values() { - return decimal64Values; - } - - /** - * Returns the high 64 bits of Decimal128 values. - */ - public long[] getDecimal128High() { - return decimal128High; - } - - /** - * Returns the low 64 bits of Decimal128 values. - */ - public long[] getDecimal128Low() { - return decimal128Low; - } - - /** - * Returns bits 255-192 of Decimal256 values. - */ - public long[] getDecimal256Hh() { - return decimal256Hh; - } - - /** - * Returns bits 191-128 of Decimal256 values. - */ - public long[] getDecimal256Hl() { - return decimal256Hl; - } - - /** - * Returns bits 127-64 of Decimal256 values. - */ - public long[] getDecimal256Lh() { - return decimal256Lh; - } - - /** - * Returns bits 63-0 of Decimal256 values. - */ - public long[] getDecimal256Ll() { - return decimal256Ll; - } - - /** - * Returns the array dimensions per row (nDims for each row). - */ - public byte[] getArrayDims() { - return arrayDims; - } - - /** - * Returns the flattened array shapes (all dimension lengths concatenated). - */ - public int[] getArrayShapes() { - return arrayShapes; - } - - /** - * Returns the current write offset in arrayShapes. - */ - public int getArrayShapeOffset() { - return arrayShapeOffset; - } - - /** - * Returns the flattened double array data. - */ - public double[] getDoubleArrayData() { - return doubleArrayData; - } - - /** - * Returns the flattened long array data. - */ - public long[] getLongArrayData() { - return longArrayData; - } - - /** - * Returns the current write offset in the data arrays. - */ - public int getArrayDataOffset() { - return arrayDataOffset; - } - - /** - * Returns the symbol indices array (one index per value). - * Each index refers to a position in the symbol dictionary. - */ - public int[] getSymbolIndices() { - return symbolIndices; - } - - /** - * Returns the symbol dictionary as a String array. - * Index i in symbolIndices maps to symbolDictionary[i]. - */ - public String[] getSymbolDictionary() { - if (symbolList == null) { - return new String[0]; - } - String[] dict = new String[symbolList.size()]; - for (int i = 0; i < symbolList.size(); i++) { - dict[i] = symbolList.get(i); - } - return dict; - } - - /** - * Returns the size of the symbol dictionary. - */ - public int getSymbolDictionarySize() { - return symbolList == null ? 0 : symbolList.size(); - } - - /** - * Returns the global symbol IDs array for delta encoding. - * Returns null if no global IDs have been stored. - */ - public int[] getGlobalSymbolIds() { - return globalSymbolIds; - } - - /** - * Returns the maximum global symbol ID used in this column. - * Returns -1 if no symbols have been added with global IDs. - */ - public int getMaxGlobalSymbolId() { - return maxGlobalSymbolId; - } - - public void addBoolean(boolean value) { - ensureCapacity(); - booleanValues[valueCount++] = value; - size++; - } - - public void addByte(byte value) { - ensureCapacity(); - byteValues[valueCount++] = value; - size++; - } - - public void addShort(short value) { - ensureCapacity(); - shortValues[valueCount++] = value; - size++; - } - - public void addInt(int value) { - ensureCapacity(); - intValues[valueCount++] = value; - size++; - } - - public void addLong(long value) { - ensureCapacity(); - longValues[valueCount++] = value; - size++; - } - - public void addFloat(float value) { - ensureCapacity(); - floatValues[valueCount++] = value; - size++; - } - - public void addDouble(double value) { - ensureCapacity(); - doubleValues[valueCount++] = value; - size++; - } - - public void addString(String value) { - ensureCapacity(); - if (value == null && nullable) { - markNull(size); - // Null strings don't take space in the value buffer - size++; - } else { - stringValues[valueCount++] = value; - size++; - } - } - - public void addSymbol(String value) { - ensureCapacity(); - if (value == null) { - if (nullable) { - markNull(size); - } - // Null symbols don't take space in the value buffer - size++; - } else { - int idx = symbolDict.get(value); - if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - idx = symbolList.size(); - symbolDict.put(value, idx); - symbolList.add(value); - } - symbolIndices[valueCount++] = idx; - size++; - } - } - - /** - * Adds a symbol with both local dictionary and global ID tracking. - * Used for delta dictionary encoding where global IDs are shared across all columns. - * - * @param value the symbol string - * @param globalId the global ID from GlobalSymbolDictionary - */ - public void addSymbolWithGlobalId(String value, int globalId) { - ensureCapacity(); - if (value == null) { - if (nullable) { - markNull(size); - } - size++; - } else { - // Add to local dictionary (for backward compatibility with existing encoder) - int localIdx = symbolDict.get(value); - if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - localIdx = symbolList.size(); - symbolDict.put(value, localIdx); - symbolList.add(value); - } - symbolIndices[valueCount] = localIdx; - - // Also store global ID for delta encoding - if (globalSymbolIds == null) { - globalSymbolIds = new int[capacity]; - } - globalSymbolIds[valueCount] = globalId; - - // Track max global ID for this column - if (globalId > maxGlobalSymbolId) { - maxGlobalSymbolId = globalId; - } - - valueCount++; - size++; - } - } - - public void addUuid(long high, long low) { - ensureCapacity(); - uuidHigh[valueCount] = high; - uuidLow[valueCount] = low; - valueCount++; - size++; - } - - public void addLong256(long l0, long l1, long l2, long l3) { - ensureCapacity(); - int offset = valueCount * 4; - long256Values[offset] = l0; - long256Values[offset + 1] = l1; - long256Values[offset + 2] = l2; - long256Values[offset + 3] = l3; - valueCount++; - size++; - } - - // ==================== Decimal methods ==================== - - /** - * Adds a Decimal64 value. - * If the value's scale differs from the column's established scale, - * the value is automatically rescaled to match. - * - * @param value the Decimal64 value to add - */ - public void addDecimal64(Decimal64 value) { - if (value == null || value.isNull()) { - addNull(); - return; - } - ensureCapacity(); - if (decimalScale == -1) { - decimalScale = (byte) value.getScale(); - decimal64Values[valueCount++] = value.getValue(); - } else if (decimalScale != value.getScale()) { - rescaleTemp.ofRaw(value.getValue()); - rescaleTemp.setScale(value.getScale()); - rescaleTemp.rescale(decimalScale); - decimal64Values[valueCount++] = rescaleTemp.getLl(); - } else { - decimal64Values[valueCount++] = value.getValue(); - } - size++; - } - - /** - * Adds a Decimal128 value. - * If the value's scale differs from the column's established scale, - * the value is automatically rescaled to match. - * - * @param value the Decimal128 value to add - */ - public void addDecimal128(Decimal128 value) { - if (value == null || value.isNull()) { - addNull(); - return; - } - ensureCapacity(); - if (decimalScale == -1) { - decimalScale = (byte) value.getScale(); - } else if (decimalScale != value.getScale()) { - rescaleTemp.ofRaw(value.getHigh(), value.getLow()); - rescaleTemp.setScale(value.getScale()); - rescaleTemp.rescale(decimalScale); - decimal128High[valueCount] = rescaleTemp.getLh(); - decimal128Low[valueCount] = rescaleTemp.getLl(); - valueCount++; - size++; - return; - } - decimal128High[valueCount] = value.getHigh(); - decimal128Low[valueCount] = value.getLow(); - valueCount++; - size++; - } - - /** - * Adds a Decimal256 value. - * If the value's scale differs from the column's established scale, - * the value is automatically rescaled to match. - * - * @param value the Decimal256 value to add - */ - public void addDecimal256(Decimal256 value) { - if (value == null || value.isNull()) { - addNull(); - return; - } - ensureCapacity(); - Decimal256 src = value; - if (decimalScale == -1) { - decimalScale = (byte) value.getScale(); - } else if (decimalScale != value.getScale()) { - rescaleTemp.copyFrom(value); - rescaleTemp.rescale(decimalScale); - src = rescaleTemp; - } - decimal256Hh[valueCount] = src.getHh(); - decimal256Hl[valueCount] = src.getHl(); - decimal256Lh[valueCount] = src.getLh(); - decimal256Ll[valueCount] = src.getLl(); - valueCount++; - size++; - } - - // ==================== Array methods ==================== - - /** - * Adds a 1D double array. - */ - public void addDoubleArray(double[] values) { - if (values == null) { - addNull(); - return; - } - ensureArrayCapacity(1, values.length); - arrayDims[valueCount] = 1; - arrayShapes[arrayShapeOffset++] = values.length; - for (double v : values) { - doubleArrayData[arrayDataOffset++] = v; - } - valueCount++; - size++; - } - - /** - * Adds a 2D double array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ - public void addDoubleArray(double[][] values) { - if (values == null) { - addNull(); - return; - } - int dim0 = values.length; - int dim1 = dim0 > 0 ? values[0].length : 0; - // Validate rectangular shape - for (int i = 1; i < dim0; i++) { - if (values[i].length != dim1) { - throw new LineSenderException("irregular array shape"); - } - } - ensureArrayCapacity(2, dim0 * dim1); - arrayDims[valueCount] = 2; - arrayShapes[arrayShapeOffset++] = dim0; - arrayShapes[arrayShapeOffset++] = dim1; - for (double[] row : values) { - for (double v : row) { - doubleArrayData[arrayDataOffset++] = v; - } - } - valueCount++; - size++; - } - - /** - * Adds a 3D double array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ public void addDoubleArray(double[][][] values) { if (values == null) { addNull(); @@ -870,7 +487,6 @@ public void addDoubleArray(double[][][] values) { int dim0 = values.length; int dim1 = dim0 > 0 ? values[0].length : 0; int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; - // Validate rectangular shape for (int i = 0; i < dim0; i++) { if (values[i].length != dim1) { throw new LineSenderException("irregular array shape"); @@ -897,16 +513,11 @@ public void addDoubleArray(double[][][] values) { size++; } - /** - * Adds a DoubleArray (N-dimensional wrapper). - * Uses a capturing approach to extract shape and data. - */ public void addDoubleArray(DoubleArray array) { if (array == null) { addNull(); return; } - // Use a capturing ArrayBufferAppender to extract the data ArrayCapture capture = new ArrayCapture(); array.appendToBufPtr(capture); @@ -922,9 +533,33 @@ public void addDoubleArray(DoubleArray array) { size++; } - /** - * Adds a 1D long array. - */ + public void addFloat(float value) { + dataBuffer.putFloat(value); + valueCount++; + size++; + } + + public void addInt(int value) { + dataBuffer.putInt(value); + valueCount++; + size++; + } + + public void addLong(long value) { + dataBuffer.putLong(value); + valueCount++; + size++; + } + + public void addLong256(long l0, long l1, long l2, long l3) { + dataBuffer.putLong(l0); + dataBuffer.putLong(l1); + dataBuffer.putLong(l2); + dataBuffer.putLong(l3); + valueCount++; + size++; + } + public void addLongArray(long[] values) { if (values == null) { addNull(); @@ -940,10 +575,6 @@ public void addLongArray(long[] values) { size++; } - /** - * Adds a 2D long array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ public void addLongArray(long[][] values) { if (values == null) { addNull(); @@ -951,7 +582,6 @@ public void addLongArray(long[][] values) { } int dim0 = values.length; int dim1 = dim0 > 0 ? values[0].length : 0; - // Validate rectangular shape for (int i = 1; i < dim0; i++) { if (values[i].length != dim1) { throw new LineSenderException("irregular array shape"); @@ -970,10 +600,6 @@ public void addLongArray(long[][] values) { size++; } - /** - * Adds a 3D long array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ public void addLongArray(long[][][] values) { if (values == null) { addNull(); @@ -982,7 +608,6 @@ public void addLongArray(long[][][] values) { int dim0 = values.length; int dim1 = dim0 > 0 ? values[0].length : 0; int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; - // Validate rectangular shape for (int i = 0; i < dim0; i++) { if (values[i].length != dim1) { throw new LineSenderException("irregular array shape"); @@ -1009,199 +634,360 @@ public void addLongArray(long[][][] values) { size++; } + public void addLongArray(LongArray array) { + if (array == null) { + addNull(); + return; + } + ArrayCapture capture = new ArrayCapture(); + array.appendToBufPtr(capture); + + ensureArrayCapacity(capture.nDims, capture.longDataOffset); + arrayDims[valueCount] = capture.nDims; + for (int i = 0; i < capture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = capture.shape[i]; + } + for (int i = 0; i < capture.longDataOffset; i++) { + longArrayData[arrayDataOffset++] = capture.longData[i]; + } + valueCount++; + size++; + } + + public void addNull() { + if (nullable) { + ensureNullCapacity(size + 1); + markNull(size); + size++; + } else { + // For non-nullable columns, store a sentinel/default value + switch (type) { + case TYPE_BOOLEAN: + dataBuffer.putByte((byte) 0); + break; + case TYPE_BYTE: + dataBuffer.putByte((byte) 0); + break; + case TYPE_SHORT: + case TYPE_CHAR: + dataBuffer.putShort((short) 0); + break; + case TYPE_INT: + dataBuffer.putInt(0); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_FLOAT: + dataBuffer.putFloat(Float.NaN); + break; + case TYPE_DOUBLE: + dataBuffer.putDouble(Double.NaN); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + ensureOnHeapCapacity(); + stringValues[valueCount] = null; + break; + case TYPE_SYMBOL: + dataBuffer.putInt(-1); + break; + case TYPE_UUID: + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_LONG256: + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_DECIMAL64: + dataBuffer.putLong(Decimals.DECIMAL64_NULL); + break; + case TYPE_DECIMAL128: + dataBuffer.putLong(Decimals.DECIMAL128_HI_NULL); + dataBuffer.putLong(Decimals.DECIMAL128_LO_NULL); + break; + case TYPE_DECIMAL256: + dataBuffer.putLong(Decimals.DECIMAL256_HH_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_HL_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); + break; + } + valueCount++; + size++; + } + } + + public void addShort(short value) { + dataBuffer.putShort(value); + valueCount++; + size++; + } + + public void addString(String value) { + if (value == null && nullable) { + ensureNullCapacity(size + 1); + markNull(size); + size++; + } else { + ensureOnHeapCapacity(); + stringValues[valueCount++] = value; + size++; + } + } + + public void addSymbol(String value) { + if (value == null) { + if (nullable) { + ensureNullCapacity(size + 1); + markNull(size); + } + size++; + } else { + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + idx = symbolList.size(); + symbolDict.put(value, idx); + symbolList.add(value); + } + dataBuffer.putInt(idx); + valueCount++; + size++; + } + } + + public void addSymbolWithGlobalId(String value, int globalId) { + if (value == null) { + if (nullable) { + ensureNullCapacity(size + 1); + markNull(size); + } + size++; + } else { + int localIdx = symbolDict.get(value); + if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + localIdx = symbolList.size(); + symbolDict.put(value, localIdx); + symbolList.add(value); + } + dataBuffer.putInt(localIdx); + + if (auxBuffer == null) { + auxBuffer = new OffHeapAppendMemory(64); + } + auxBuffer.putInt(globalId); + + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; + } + + valueCount++; + size++; + } + } + + public void addUuid(long high, long low) { + // Store in wire order: lo first, hi second + dataBuffer.putLong(low); + dataBuffer.putLong(high); + valueCount++; + size++; + } + + @Override + public void close() { + if (dataBuffer != null) { + dataBuffer.close(); + dataBuffer = null; + } + if (auxBuffer != null) { + auxBuffer.close(); + auxBuffer = null; + } + if (nullBufPtr != 0) { + Unsafe.free(nullBufPtr, (long) nullBufCapRows >>> 3, MemoryTag.NATIVE_ILP_RSS); + nullBufPtr = 0; + nullBufCapRows = 0; + } + } + + public int getArrayDataOffset() { + return arrayDataOffset; + } + + public byte[] getArrayDims() { + return arrayDims; + } + + public int[] getArrayShapes() { + return arrayShapes; + } + + public int getArrayShapeOffset() { + return arrayShapeOffset; + } + + /** + * Returns the off-heap address of the auxiliary data buffer (global symbol IDs). + * Returns 0 if no auxiliary data exists. + */ + public long getAuxDataAddress() { + return auxBuffer != null ? auxBuffer.pageAddress() : 0; + } + + /** + * Returns the off-heap address of the column data buffer. + */ + public long getDataAddress() { + return dataBuffer != null ? dataBuffer.pageAddress() : 0; + } + + /** + * Returns the number of bytes of data in the off-heap buffer. + */ + public long getDataSize() { + return dataBuffer != null ? dataBuffer.getAppendOffset() : 0; + } + + public byte getDecimalScale() { + return decimalScale; + } + + public double[] getDoubleArrayData() { + return doubleArrayData; + } + + public long[] getLongArrayData() { + return longArrayData; + } + + public int getMaxGlobalSymbolId() { + return maxGlobalSymbolId; + } + + public String getName() { + return name; + } + /** - * Adds a LongArray (N-dimensional wrapper). - * Uses a capturing approach to extract shape and data. + * Returns the off-heap address of the null bitmap. + * Returns 0 for non-nullable columns. */ - public void addLongArray(LongArray array) { - if (array == null) { - addNull(); - return; - } - // Use a capturing ArrayBufferAppender to extract the data - ArrayCapture capture = new ArrayCapture(); - array.appendToBufPtr(capture); + public long getNullBitmapAddress() { + return nullBufPtr; + } - ensureArrayCapacity(capture.nDims, capture.longDataOffset); - arrayDims[valueCount] = capture.nDims; - for (int i = 0; i < capture.nDims; i++) { - arrayShapes[arrayShapeOffset++] = capture.shape[i]; + /** + * Returns the bit-packed null bitmap as a long array. + * This creates a new array from off-heap data. + */ + public long[] getNullBitmapPacked() { + if (nullBufPtr == 0) { + return null; } - for (int i = 0; i < capture.longDataOffset; i++) { - longArrayData[arrayDataOffset++] = capture.longData[i]; + int longCount = (size + 63) >>> 6; + long[] result = new long[longCount]; + for (int i = 0; i < longCount; i++) { + result[i] = Unsafe.getUnsafe().getLong(nullBufPtr + (long) i * 8); } - valueCount++; - size++; + return result; } - /** - * Ensures capacity for array storage. - * @param nDims number of dimensions for this array - * @param dataElements number of data elements - */ - private void ensureArrayCapacity(int nDims, int dataElements) { - ensureCapacity(); // For row-level capacity (arrayDims uses valueCount) + public int getSize() { + return size; + } - // Ensure shape array capacity - int requiredShapeCapacity = arrayShapeOffset + nDims; - if (arrayShapes == null) { - arrayShapes = new int[Math.max(64, requiredShapeCapacity)]; - } else if (requiredShapeCapacity > arrayShapes.length) { - arrayShapes = Arrays.copyOf(arrayShapes, Math.max(arrayShapes.length * 2, requiredShapeCapacity)); - } + public String[] getStringValues() { + return stringValues; + } - // Ensure data array capacity - int requiredDataCapacity = arrayDataOffset + dataElements; - if (type == TYPE_DOUBLE_ARRAY) { - if (doubleArrayData == null) { - doubleArrayData = new double[Math.max(256, requiredDataCapacity)]; - } else if (requiredDataCapacity > doubleArrayData.length) { - doubleArrayData = Arrays.copyOf(doubleArrayData, Math.max(doubleArrayData.length * 2, requiredDataCapacity)); - } - } else if (type == TYPE_LONG_ARRAY) { - if (longArrayData == null) { - longArrayData = new long[Math.max(256, requiredDataCapacity)]; - } else if (requiredDataCapacity > longArrayData.length) { - longArrayData = Arrays.copyOf(longArrayData, Math.max(longArrayData.length * 2, requiredDataCapacity)); - } + public String[] getSymbolDictionary() { + if (symbolList == null) { + return new String[0]; } + String[] dict = new String[symbolList.size()]; + for (int i = 0; i < symbolList.size(); i++) { + dict[i] = symbolList.get(i); + } + return dict; } - public void addNull() { - ensureCapacity(); - if (nullable) { - // For nullable columns, mark null in bitmap but don't store a value - markNull(size); - size++; - } else { - // For non-nullable columns, we must store a sentinel/default value - // because no null bitmap will be written - switch (type) { - case TYPE_BOOLEAN: - booleanValues[valueCount++] = false; - break; - case TYPE_BYTE: - byteValues[valueCount++] = 0; - break; - case TYPE_SHORT: - case TYPE_CHAR: - shortValues[valueCount++] = 0; - break; - case TYPE_INT: - intValues[valueCount++] = 0; - break; - case TYPE_LONG: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: - longValues[valueCount++] = Long.MIN_VALUE; - break; - case TYPE_FLOAT: - floatValues[valueCount++] = Float.NaN; - break; - case TYPE_DOUBLE: - doubleValues[valueCount++] = Double.NaN; - break; - case TYPE_STRING: - case TYPE_VARCHAR: - stringValues[valueCount++] = null; - break; - case TYPE_SYMBOL: - symbolIndices[valueCount++] = -1; - break; - case TYPE_UUID: - uuidHigh[valueCount] = Long.MIN_VALUE; - uuidLow[valueCount] = Long.MIN_VALUE; - valueCount++; - break; - case TYPE_LONG256: - int offset = valueCount * 4; - long256Values[offset] = Long.MIN_VALUE; - long256Values[offset + 1] = Long.MIN_VALUE; - long256Values[offset + 2] = Long.MIN_VALUE; - long256Values[offset + 3] = Long.MIN_VALUE; - valueCount++; - break; - case TYPE_DECIMAL64: - decimal64Values[valueCount++] = Decimals.DECIMAL64_NULL; - break; - case TYPE_DECIMAL128: - decimal128High[valueCount] = Decimals.DECIMAL128_HI_NULL; - decimal128Low[valueCount] = Decimals.DECIMAL128_LO_NULL; - valueCount++; - break; - case TYPE_DECIMAL256: - decimal256Hh[valueCount] = Decimals.DECIMAL256_HH_NULL; - decimal256Hl[valueCount] = Decimals.DECIMAL256_HL_NULL; - decimal256Lh[valueCount] = Decimals.DECIMAL256_LH_NULL; - decimal256Ll[valueCount] = Decimals.DECIMAL256_LL_NULL; - valueCount++; - break; - } - size++; - } + public int getSymbolDictionarySize() { + return symbolList == null ? 0 : symbolList.size(); } - private void markNull(int index) { - int longIndex = index >>> 6; + public byte getType() { + return type; + } + + public int getValueCount() { + return valueCount; + } + + public boolean hasNulls() { + return hasNulls; + } + + public boolean isNull(int index) { + if (nullBufPtr == 0) { + return false; + } + long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; int bitIndex = index & 63; - nullBitmapPacked[longIndex] |= (1L << bitIndex); - hasNulls = true; + return (Unsafe.getUnsafe().getLong(longAddr) & (1L << bitIndex)) != 0; } public void reset() { size = 0; valueCount = 0; hasNulls = false; - if (nullBitmapPacked != null) { - Arrays.fill(nullBitmapPacked, 0L); + if (dataBuffer != null) { + dataBuffer.truncate(); + } + if (auxBuffer != null) { + auxBuffer.truncate(); + } + if (nullBufPtr != 0) { + Vect.memset(nullBufPtr, (long) nullBufCapRows >>> 3, 0); } if (symbolDict != null) { symbolDict.clear(); symbolList.clear(); } - // Reset global symbol tracking maxGlobalSymbolId = -1; - // Reset array tracking arrayShapeOffset = 0; arrayDataOffset = 0; - // Reset decimal scale (will be set by first non-null value) decimalScale = -1; } - /** - * Truncates the column to the specified size. - * This is used to cancel uncommitted row values. - * - * @param newSize the target size (number of rows) - */ public void truncateTo(int newSize) { if (newSize >= size) { - return; // Nothing to truncate + return; } - // Count non-null values up to newSize int newValueCount = 0; - if (nullable && nullBitmapPacked != null) { + if (nullable && nullBufPtr != 0) { for (int i = 0; i < newSize; i++) { - int longIndex = i >>> 6; - int bitIndex = i & 63; - if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) == 0) { + if (!isNull(i)) { newValueCount++; } } // Clear null bits for truncated rows for (int i = newSize; i < size; i++) { - int longIndex = i >>> 6; + long longAddr = nullBufPtr + ((long) (i >>> 6)) * 8; int bitIndex = i & 63; - nullBitmapPacked[longIndex] &= ~(1L << bitIndex); + long current = Unsafe.getUnsafe().getLong(longAddr); + Unsafe.getUnsafe().putLong(longAddr, current & ~(1L << bitIndex)); } - // Recompute hasNulls hasNulls = false; for (int i = 0; i < newSize && !hasNulls; i++) { - int longIndex = i >>> 6; - int bitIndex = i & 63; - if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0) { + if (isNull(i)) { hasNulls = true; } } @@ -1211,223 +997,197 @@ public void truncateTo(int newSize) { size = newSize; valueCount = newValueCount; - } - private void ensureCapacity() { - if (size >= capacity) { - int newCapacity = capacity * 2; - growStorage(type, newCapacity); - if (nullable && nullBitmapPacked != null) { - int newLongCount = (newCapacity + 63) >>> 6; - nullBitmapPacked = Arrays.copyOf(nullBitmapPacked, newLongCount); - } - capacity = newCapacity; + // Rewind off-heap data buffer + if (dataBuffer != null && elemSize > 0) { + dataBuffer.jumpTo((long) newValueCount * elemSize); + } + + // Rewind aux buffer (symbol global IDs) + if (auxBuffer != null) { + auxBuffer.jumpTo((long) newValueCount * 4); } } - private void allocateStorage(byte type, int cap) { + private void allocateStorage(byte type) { switch (type) { case TYPE_BOOLEAN: - booleanValues = new boolean[cap]; - break; case TYPE_BYTE: - byteValues = new byte[cap]; + dataBuffer = new OffHeapAppendMemory(16); break; case TYPE_SHORT: case TYPE_CHAR: - shortValues = new short[cap]; + dataBuffer = new OffHeapAppendMemory(32); break; case TYPE_INT: - intValues = new int[cap]; + dataBuffer = new OffHeapAppendMemory(64); break; case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: case TYPE_DATE: - longValues = new long[cap]; + dataBuffer = new OffHeapAppendMemory(128); break; case TYPE_FLOAT: - floatValues = new float[cap]; + dataBuffer = new OffHeapAppendMemory(64); break; case TYPE_DOUBLE: - doubleValues = new double[cap]; + dataBuffer = new OffHeapAppendMemory(128); break; case TYPE_STRING: case TYPE_VARCHAR: - stringValues = new String[cap]; + stringValues = new String[onHeapCapacity]; break; case TYPE_SYMBOL: - symbolIndices = new int[cap]; + dataBuffer = new OffHeapAppendMemory(64); symbolDict = new CharSequenceIntHashMap(); symbolList = new ObjList<>(); break; case TYPE_UUID: - uuidHigh = new long[cap]; - uuidLow = new long[cap]; + dataBuffer = new OffHeapAppendMemory(256); break; case TYPE_LONG256: - // Flat array: 4 longs per value - long256Values = new long[cap * 4]; + dataBuffer = new OffHeapAppendMemory(512); break; case TYPE_DOUBLE_ARRAY: case TYPE_LONG_ARRAY: - // Array types: allocate per-row tracking - // Shape and data arrays are grown dynamically in ensureArrayCapacity() - arrayDims = new byte[cap]; - arrayRowCapacity = cap; + arrayDims = new byte[onHeapCapacity]; break; case TYPE_DECIMAL64: - decimal64Values = new long[cap]; + dataBuffer = new OffHeapAppendMemory(128); break; case TYPE_DECIMAL128: - decimal128High = new long[cap]; - decimal128Low = new long[cap]; + dataBuffer = new OffHeapAppendMemory(256); break; case TYPE_DECIMAL256: - decimal256Hh = new long[cap]; - decimal256Hl = new long[cap]; - decimal256Lh = new long[cap]; - decimal256Ll = new long[cap]; + dataBuffer = new OffHeapAppendMemory(512); break; } } - private void growStorage(byte type, int newCap) { - switch (type) { - case TYPE_BOOLEAN: - booleanValues = Arrays.copyOf(booleanValues, newCap); - break; - case TYPE_BYTE: - byteValues = Arrays.copyOf(byteValues, newCap); - break; - case TYPE_SHORT: - case TYPE_CHAR: - shortValues = Arrays.copyOf(shortValues, newCap); - break; - case TYPE_INT: - intValues = Arrays.copyOf(intValues, newCap); - break; - case TYPE_LONG: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: - longValues = Arrays.copyOf(longValues, newCap); - break; - case TYPE_FLOAT: - floatValues = Arrays.copyOf(floatValues, newCap); - break; - case TYPE_DOUBLE: - doubleValues = Arrays.copyOf(doubleValues, newCap); - break; - case TYPE_STRING: - case TYPE_VARCHAR: - stringValues = Arrays.copyOf(stringValues, newCap); - break; - case TYPE_SYMBOL: - symbolIndices = Arrays.copyOf(symbolIndices, newCap); - if (globalSymbolIds != null) { - globalSymbolIds = Arrays.copyOf(globalSymbolIds, newCap); - } - break; - case TYPE_UUID: - uuidHigh = Arrays.copyOf(uuidHigh, newCap); - uuidLow = Arrays.copyOf(uuidLow, newCap); - break; - case TYPE_LONG256: - // Flat array: 4 longs per value - long256Values = Arrays.copyOf(long256Values, newCap * 4); - break; - case TYPE_DOUBLE_ARRAY: - case TYPE_LONG_ARRAY: - // Array types: grow per-row tracking - arrayDims = Arrays.copyOf(arrayDims, newCap); - arrayRowCapacity = newCap; - // Note: shapes and data arrays are grown in ensureArrayCapacity() - break; - case TYPE_DECIMAL64: - decimal64Values = Arrays.copyOf(decimal64Values, newCap); - break; - case TYPE_DECIMAL128: - decimal128High = Arrays.copyOf(decimal128High, newCap); - decimal128Low = Arrays.copyOf(decimal128Low, newCap); - break; - case TYPE_DECIMAL256: - decimal256Hh = Arrays.copyOf(decimal256Hh, newCap); - decimal256Hl = Arrays.copyOf(decimal256Hl, newCap); - decimal256Lh = Arrays.copyOf(decimal256Lh, newCap); - decimal256Ll = Arrays.copyOf(decimal256Ll, newCap); - break; + private void ensureArrayCapacity(int nDims, int dataElements) { + // Ensure per-row array dims capacity + if (valueCount >= arrayDims.length) { + arrayDims = Arrays.copyOf(arrayDims, arrayDims.length * 2); + } + + // Ensure null bitmap capacity + if (nullable) { + ensureNullCapacity(size + 1); + } + + // Ensure shape array capacity + int requiredShapeCapacity = arrayShapeOffset + nDims; + if (arrayShapes == null) { + arrayShapes = new int[Math.max(64, requiredShapeCapacity)]; + } else if (requiredShapeCapacity > arrayShapes.length) { + arrayShapes = Arrays.copyOf(arrayShapes, Math.max(arrayShapes.length * 2, requiredShapeCapacity)); + } + + // Ensure data array capacity + int requiredDataCapacity = arrayDataOffset + dataElements; + if (type == TYPE_DOUBLE_ARRAY) { + if (doubleArrayData == null) { + doubleArrayData = new double[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > doubleArrayData.length) { + doubleArrayData = Arrays.copyOf(doubleArrayData, Math.max(doubleArrayData.length * 2, requiredDataCapacity)); + } + } else if (type == TYPE_LONG_ARRAY) { + if (longArrayData == null) { + longArrayData = new long[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > longArrayData.length) { + longArrayData = Arrays.copyOf(longArrayData, Math.max(longArrayData.length * 2, requiredDataCapacity)); + } } } + + private void ensureNullCapacity(int rows) { + if (rows > nullBufCapRows) { + int newCapRows = Math.max(nullBufCapRows * 2, ((rows + 63) >>> 6) << 6); + long newSizeBytes = (long) newCapRows >>> 3; + long oldSizeBytes = (long) nullBufCapRows >>> 3; + nullBufPtr = Unsafe.realloc(nullBufPtr, oldSizeBytes, newSizeBytes, MemoryTag.NATIVE_ILP_RSS); + Vect.memset(nullBufPtr + oldSizeBytes, newSizeBytes - oldSizeBytes, 0); + nullBufCapRows = newCapRows; + } + } + + private void ensureOnHeapCapacity() { + if (valueCount >= onHeapCapacity) { + int newCapacity = onHeapCapacity * 2; + if (stringValues != null) { + stringValues = Arrays.copyOf(stringValues, newCapacity); + } + onHeapCapacity = newCapacity; + } + } + + private void markNull(int index) { + long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; + int bitIndex = index & 63; + long current = Unsafe.getUnsafe().getLong(longAddr); + Unsafe.getUnsafe().putLong(longAddr, current | (1L << bitIndex)); + hasNulls = true; + } } /** * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr(). - * This implements ArrayBufferAppender to intercept the serialization and extract - * shape and data into Java arrays for storage in ColumnBuffer. */ private static class ArrayCapture implements ArrayBufferAppender { byte nDims; - int[] shape = new int[32]; // Max 32 dimensions + int[] shape = new int[32]; int shapeIndex; double[] doubleData; int doubleDataOffset; long[] longData; int longDataOffset; + @Override + public void putBlockOfBytes(long from, long len) { + int count = (int) (len / 8); + if (doubleData == null) { + doubleData = new double[count]; + } + for (int i = 0; i < count; i++) { + doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L); + } + } + @Override public void putByte(byte b) { if (shapeIndex == 0) { - // First byte is nDims nDims = b; } } + @Override + public void putDouble(double value) { + if (doubleData != null && doubleDataOffset < doubleData.length) { + doubleData[doubleDataOffset++] = value; + } + } + @Override public void putInt(int value) { - // Shape dimensions if (shapeIndex < nDims) { shape[shapeIndex++] = value; - // Once we have all dimensions, compute total elements and allocate data array if (shapeIndex == nDims) { int totalElements = 1; for (int i = 0; i < nDims; i++) { totalElements *= shape[i]; } - // Allocate both - only one will be used doubleData = new double[totalElements]; longData = new long[totalElements]; } } } - @Override - public void putDouble(double value) { - if (doubleData != null && doubleDataOffset < doubleData.length) { - doubleData[doubleDataOffset++] = value; - } - } - @Override public void putLong(long value) { if (longData != null && longDataOffset < longData.length) { longData[longDataOffset++] = value; } } - - @Override - public void putBlockOfBytes(long from, long len) { - // This is the bulk data from the array - // The AbstractArray uses this to copy raw bytes - // We need to figure out if it's doubles or longs based on context - // For now, assume doubles (8 bytes each) since DoubleArray uses this - int count = (int) (len / 8); - if (doubleData == null) { - doubleData = new double[count]; - } - for (int i = 0; i < count; i++) { - doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L); - } - } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java new file mode 100644 index 0000000..96c39a3 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -0,0 +1,266 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class OffHeapAppendMemoryTest { + + @Test + public void testPutAndReadByte() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 42); + mem.putByte((byte) -1); + mem.putByte((byte) 0); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testPutAndReadShort() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putShort((short) 12_345); + mem.putShort(Short.MIN_VALUE); + mem.putShort(Short.MAX_VALUE); + + assertEquals(6, mem.getAppendOffset()); + assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); + assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); + assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); + } + } + + @Test + public void testPutAndReadInt() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(100_000); + mem.putInt(Integer.MIN_VALUE); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); + } + } + + @Test + public void testPutAndReadLong() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(1_000_000_000_000L); + mem.putLong(Long.MIN_VALUE); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + } + + @Test + public void testPutAndReadFloat() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putFloat(3.14f); + mem.putFloat(Float.NaN); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); + assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); + } + } + + @Test + public void testPutAndReadDouble() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putDouble(2.718281828); + mem.putDouble(Double.NaN); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); + } + } + + @Test + public void testPutBoolean() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putBoolean(true); + mem.putBoolean(false); + mem.putBoolean(true); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + } + + @Test + public void testGrowth() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write more data than initial capacity to force growth + for (int i = 0; i < 100; i++) { + mem.putLong(i); + } + + assertEquals(800, mem.getAppendOffset()); + for (int i = 0; i < 100; i++) { + assertEquals(i, Unsafe.getUnsafe().getLong(mem.addressOf((long) i * 8))); + } + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testTruncate() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.putInt(2); + mem.putInt(3); + assertEquals(12, mem.getAppendOffset()); + + mem.truncate(); + assertEquals(0, mem.getAppendOffset()); + + // Can write again after truncate + mem.putInt(42); + assertEquals(4, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + } + } + + @Test + public void testJumpTo() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(100); + mem.putLong(200); + mem.putLong(300); + assertEquals(24, mem.getAppendOffset()); + + // Jump back to offset 8 (after first long) + mem.jumpTo(8); + assertEquals(8, mem.getAppendOffset()); + + // Write new value at offset 8 + mem.putLong(999); + assertEquals(16, mem.getAppendOffset()); + assertEquals(100, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(999, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + } + + @Test + public void testSkip() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.skip(8); + mem.putInt(2); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + } + } + + @Test + public void testPageAddress() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + assertTrue(mem.pageAddress() != 0); + assertEquals(mem.pageAddress(), mem.addressOf(0)); + mem.putLong(42); + assertEquals(mem.pageAddress() + 8, mem.addressOf(8)); + } + } + + @Test + public void testCloseFreesMemory() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); + long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertTrue(during > before); + + mem.close(); + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testDoubleCloseIsSafe() { + OffHeapAppendMemory mem = new OffHeapAppendMemory(); + mem.putInt(42); + mem.close(); + mem.close(); // should not throw + } + + @Test + public void testMixedTypes() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 1); + mem.putShort((short) 2); + mem.putInt(3); + mem.putLong(4L); + mem.putFloat(5.0f); + mem.putDouble(6.0); + + long addr = mem.pageAddress(); + assertEquals(1, Unsafe.getUnsafe().getByte(addr)); + assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); + assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); + assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); + assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); + assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); + assertEquals(27, mem.getAppendOffset()); + } + } + + @Test + public void testLargeGrowth() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write 10000 doubles to stress growth + for (int i = 0; i < 10_000; i++) { + mem.putDouble(i * 1.1); + } + assertEquals(80_000, mem.getAppendOffset()); + + // Verify first and last values + assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } +} From 7f16328b1bdb03e946a1d515a02a38248711897b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 22 Feb 2026 03:14:36 +0000 Subject: [PATCH 014/230] wip 12 --- .../qwp/client/QwpWebSocketEncoder.java | 17 +- .../qwp/client/QwpWebSocketSender.java | 1504 ++++++++--------- .../qwp/protocol/QwpGorillaEncoder.java | 47 +- 3 files changed, 781 insertions(+), 787 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ef473de..8f8d2ac 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -567,38 +567,29 @@ private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long /** * Writes a timestamp column with optional Gorilla compression. - * Reads longs from off-heap. For Gorilla encoding, creates a temporary - * on-heap array since the Gorilla encoder requires long[]. + * Reads longs directly from off-heap — zero heap allocation. */ private void writeTimestampColumn(long addr, int count, boolean useGorilla) { if (useGorilla && count > 2) { - // Extract to temp array for Gorilla encoder (which requires long[]) - long[] values = new long[count]; - for (int i = 0; i < count; i++) { - values[i] = Unsafe.getUnsafe().getLong(addr + (long) i * 8); - } - - if (QwpGorillaEncoder.canUseGorilla(values, count)) { + if (QwpGorillaEncoder.canUseGorilla(addr, count)) { buffer.putByte(ENCODING_GORILLA); - int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(addr, count); buffer.ensureCapacity(encodedSize); int bytesWritten = gorillaEncoder.encodeTimestamps( buffer.getBufferPtr() + buffer.getPosition(), buffer.getCapacity() - buffer.getPosition(), - values, + addr, count ); buffer.skip(bytesWritten); } else { buffer.putByte(ENCODING_UNCOMPRESSED); - // Bulk copy for uncompressed path buffer.putBlockOfBytes(addr, (long) count * 8); } } else { if (useGorilla) { buffer.putByte(ENCODING_UNCOMPRESSED); } - // Bulk copy for uncompressed timestamps buffer.putBlockOfBytes(addr, (long) count * 8); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index dabda70..68dc19d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -24,27 +24,25 @@ package io.questdb.client.cutlass.qwp.client; -import io.questdb.client.cutlass.qwp.protocol.*; - import io.questdb.client.Sender; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketClientFactory; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; - import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import io.questdb.client.std.Chars; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; -import io.questdb.client.std.LongHashSet; -import io.questdb.client.std.ObjList; +import io.questdb.client.std.Chars; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.LongHashSet; +import io.questdb.client.std.ObjList; import io.questdb.client.std.bytes.DirectByteSlice; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -87,81 +85,74 @@ */ public class QwpWebSocketSender implements Sender { - private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); - - private static final int DEFAULT_BUFFER_SIZE = 8192; - private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB - public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; public static final int DEFAULT_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms + public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 public static final int DEFAULT_SEND_QUEUE_CAPACITY = WebSocketSendQueue.DEFAULT_QUEUE_CAPACITY; // 16 + private static final int DEFAULT_BUFFER_SIZE = 8192; + private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB + private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final String WRITE_PATH = "/write/v4"; - + private final int autoFlushBytes; + private final long autoFlushIntervalNanos; + // Auto-flush configuration + private final int autoFlushRows; + // Encoder for ILP v4 messages + private final QwpWebSocketEncoder encoder; + // Global symbol dictionary for delta encoding + private final GlobalSymbolDictionary globalSymbolDictionary; private final String host; + // Flow control configuration + private final int inFlightWindowSize; private final int port; - private final boolean tlsEnabled; + private final int sendQueueCapacity; + // Track schema hashes that have been sent to the server (for schema reference mode) + // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. + // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. + private final LongHashSet sentSchemaHashes = new LongHashSet(); private final CharSequenceObjHashMap tableBuffers; - private QwpTableBuffer currentTableBuffer; - private String currentTableName; + private final boolean tlsEnabled; + private MicrobatchBuffer activeBuffer; + // Double-buffering for async I/O + private MicrobatchBuffer buffer0; + private MicrobatchBuffer buffer1; // Cached column references to avoid repeated hashmap lookups private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; - - // Encoder for ILP v4 messages - private final QwpWebSocketEncoder encoder; - // WebSocket client (zero-GC native implementation) private WebSocketClient client; - private boolean connected; private boolean closed; - - // Double-buffering for async I/O - private MicrobatchBuffer buffer0; - private MicrobatchBuffer buffer1; - private MicrobatchBuffer activeBuffer; - private WebSocketSendQueue sendQueue; - - // Flow control - private InFlightWindow inFlightWindow; - - // Auto-flush configuration - private final int autoFlushRows; - private final int autoFlushBytes; - private final long autoFlushIntervalNanos; - - // Flow control configuration - private final int inFlightWindowSize; - private final int sendQueueCapacity; - - // Configuration - private boolean gorillaEnabled = true; - - // Async mode: pending row tracking - private int pendingRowCount; - private long firstPendingRowTimeNanos; - - // Batch sequence counter (must match server's messageSequence) - private long nextBatchSequence = 0; - - // Global symbol dictionary for delta encoding - private final GlobalSymbolDictionary globalSymbolDictionary; - + private boolean connected; // Track max global symbol ID used in current batch (for delta calculation) private int currentBatchMaxSymbolId = -1; - + private QwpTableBuffer currentTableBuffer; + private String currentTableName; + private long firstPendingRowTimeNanos; + // Configuration + private boolean gorillaEnabled = true; + // Flow control + private InFlightWindow inFlightWindow; // Track highest symbol ID sent to server (for delta encoding) // Once sent over TCP, server is guaranteed to receive it (or connection dies) private volatile int maxSentSymbolId = -1; + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + // Async mode: pending row tracking + private int pendingRowCount; + private WebSocketSendQueue sendQueue; - // Track schema hashes that have been sent to the server (for schema reference mode) - // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. - // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. - private final LongHashSet sentSchemaHashes = new LongHashSet(); - - private QwpWebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, - int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, int sendQueueCapacity) { + private QwpWebSocketSender( + String host, + int port, + boolean tlsEnabled, + int bufferSize, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + int sendQueueCapacity + ) { this.host = host; this.port = port; this.tlsEnabled = tlsEnabled; @@ -223,17 +214,17 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl /** * Creates a new sender with async mode and custom configuration. * - * @param host server host - * @param port server HTTP port - * @param tlsEnabled whether to use TLS - * @param autoFlushRows rows per batch (0 = no limit) - * @param autoFlushBytes bytes per batch (0 = no limit) - * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @return connected sender */ public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos) { + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos) { return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, DEFAULT_IN_FLIGHT_WINDOW_SIZE, DEFAULT_SEND_QUEUE_CAPACITY); } @@ -241,20 +232,26 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls /** * Creates a new sender with async mode and full configuration including flow control. * - * @param host server host - * @param port server HTTP port - * @param tlsEnabled whether to use TLS - * @param autoFlushRows rows per batch (0 = no limit) - * @param autoFlushBytes bytes per batch (0 = no limit) - * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) - * @param inFlightWindowSize max batches awaiting server ACK (default: 8) - * @param sendQueueCapacity max batches waiting to send (default: 16) + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize max batches awaiting server ACK (default: 8) + * @param sendQueueCapacity max batches waiting to send (default: 16) * @return connected sender */ - public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos, - int inFlightWindowSize, int sendQueueCapacity) { + public static QwpWebSocketSender connectAsync( + String host, + int port, + boolean tlsEnabled, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + int sendQueueCapacity + ) { QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, @@ -304,8 +301,8 @@ public static QwpWebSocketSender create( *

* This allows unit tests to test sender logic without requiring a real server. * - * @param host server host (not connected) - * @param port server port (not connected) + * @param host server host (not connected) + * @param port server port (not connected) * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async * @return unconnected sender */ @@ -342,122 +339,163 @@ public static QwpWebSocketSender createForTesting( // Note: does NOT call ensureConnected() } - private void ensureConnected() { - if (closed) { - throw new LineSenderException("Sender is closed"); + @Override + public void at(long timestamp, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + atNanos(timestamp); + } else { + long micros = toMicros(timestamp, unit); + atMicros(micros); } - if (!connected) { - // Create WebSocket client using factory (zero-GC native implementation) - if (tlsEnabled) { - client = WebSocketClientFactory.newInsecureTlsInstance(); - } else { - client = WebSocketClientFactory.newPlainTextInstance(); - } - - // Connect and upgrade to WebSocket - try { - client.connect(host, port); - client.upgrade(WRITE_PATH); - } catch (Exception e) { - client.close(); - client = null; - throw new LineSenderException("Failed to connect to " + host + ":" + port, e); - } - - // a window for tracking batches awaiting ACK (both modes) - inFlightWindow = new InFlightWindow(inFlightWindowSize, InFlightWindow.DEFAULT_TIMEOUT_MS); - - // Initialize send queue for async mode (window > 1) - // The send queue handles both sending AND receiving (single I/O thread) - if (inFlightWindowSize > 1) { - sendQueue = new WebSocketSendQueue(client, inFlightWindow, - sendQueueCapacity, - WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, - WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); - } - // Sync mode (window=1): no send queue - we send and read ACKs synchronously - - // Clear sent schema hashes - server starts fresh on each connection - sentSchemaHashes.clear(); + } - connected = true; - LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}]", host, port, inFlightWindowSize); - } + @Override + public void at(Instant timestamp) { + checkNotClosed(); + checkTableSelected(); + long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; + atMicros(micros); } - /** - * Returns whether Gorilla encoding is enabled. - */ - public boolean isGorillaEnabled() { - return gorillaEnabled; + @Override + public void atNow() { + checkNotClosed(); + checkTableSelected(); + // Server-assigned timestamp - just send the row without designated timestamp + sendRow(); } - /** - * Sets whether to use Gorilla timestamp encoding. - */ - public QwpWebSocketSender setGorillaEnabled(boolean enabled) { - this.gorillaEnabled = enabled; - this.encoder.setGorillaEnabled(enabled); + @Override + public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + col.addBoolean(value); return this; } - /** - * Returns whether async mode is enabled (window size > 1). - */ - public boolean isAsyncMode() { - return inFlightWindowSize > 1; + @Override + public DirectByteSlice bufferView() { + throw new LineSenderException("bufferView() is not supported for WebSocket sender"); } /** - * Returns the in-flight window size. - * Window=1 means sync mode, window>1 means async mode. + * Adds a BYTE column value to the current row. + * + * @param columnName the column name + * @param value the byte value + * @return this sender for method chaining */ - public int getInFlightWindowSize() { - return inFlightWindowSize; + public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false); + col.addByte(value); + return this; } - /** - * Returns the send queue capacity. - */ - public int getSendQueueCapacity() { - return sendQueueCapacity; + @Override + public void cancelRow() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + } } /** - * Returns the auto-flush row threshold. + * Adds a CHAR column value to the current row. + *

+ * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB. + * + * @param columnName the column name + * @param value the character value + * @return this sender for method chaining */ - public int getAutoFlushRows() { - return autoFlushRows; + public QwpWebSocketSender charColumn(CharSequence columnName, char value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); + col.addShort((short) value); + return this; } - /** - * Returns the auto-flush byte threshold. - */ - public int getAutoFlushBytes() { - return autoFlushBytes; - } + @Override + public void close() { + if (!closed) { + closed = true; - /** - * Returns the auto-flush interval in nanoseconds. - */ - public long getAutoFlushIntervalNanos() { - return autoFlushIntervalNanos; + // Flush any remaining data + try { + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush accumulated rows in table buffers first + flushPendingRows(); + + if (activeBuffer != null && activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + if (sendQueue != null) { + sendQueue.close(); + } + } else { + // Sync mode (window=1): flush pending rows synchronously + if (pendingRowCount > 0 && client != null && client.isConnected()) { + flushSync(); + } + } + } catch (Exception e) { + LOG.error("Error during close: {}", String.valueOf(e)); + } + + // Close buffers (async mode only, window > 1) + if (buffer0 != null) { + buffer0.close(); + } + if (buffer1 != null) { + buffer1.close(); + } + + if (client != null) { + client.close(); + client = null; + } + encoder.close(); + // Close all table buffers to free off-heap column memory + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + tb.close(); + } + } + } + tableBuffers.clear(); + + LOG.info("QwpWebSocketSender closed"); + } } - /** - * Returns the global symbol dictionary. - * For testing and encoder integration. - */ - public GlobalSymbolDictionary getGlobalSymbolDictionary() { - return globalSymbolDictionary; + @Override + public Sender decimalColumn(CharSequence name, Decimal64 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + col.addDecimal64(value); + return this; } - /** - * Returns the max symbol ID sent to the server. - * Once sent over TCP, server is guaranteed to receive it (or connection dies). - */ - public int getMaxSentSymbolId() { - return maxSentSymbolId; + @Override + public Sender decimalColumn(CharSequence name, Decimal128 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + col.addDecimal128(value); + return this; } // ==================== Fast-path API for high-throughput generators ==================== @@ -479,143 +517,71 @@ public int getMaxSentSymbolId() { // tableBuffer.nextRow(); // sender.incrementPendingRowCount(); - /** - * Gets or creates a table buffer for direct access. - * For high-throughput generators that want to bypass fluent API overhead. - */ - public QwpTableBuffer getTableBuffer(String tableName) { - QwpTableBuffer buffer = tableBuffers.get(tableName); - if (buffer == null) { - buffer = new QwpTableBuffer(tableName); - tableBuffers.put(tableName, buffer); - } - currentTableBuffer = buffer; - currentTableName = tableName; - return buffer; - } - - /** - * Registers a symbol in the global dictionary and returns its ID. - * For use with fast-path column buffer access. - */ - public int getOrAddGlobalSymbol(String value) { - int globalId = globalSymbolDictionary.getOrAddSymbol(value); - if (globalId > currentBatchMaxSymbolId) { - currentBatchMaxSymbolId = globalId; - } - return globalId; - } - - /** - * Increments the pending row count for auto-flush tracking. - * Call this after adding a complete row via fast-path API. - * Triggers auto-flush if any threshold is exceeded. - */ - public void incrementPendingRowCount() { - if (pendingRowCount == 0) { - firstPendingRowTimeNanos = System.nanoTime(); - } - pendingRowCount++; - - // Check if any flush threshold is exceeded (same as sendRow()) - if (shouldAutoFlush()) { - if (inFlightWindowSize > 1) { - flushPendingRows(); - } else { - // Sync mode (window=1): flush directly with ACK wait - flushSync(); - } - } - } - - // ==================== Sender interface implementation ==================== - @Override - public QwpWebSocketSender table(CharSequence tableName) { + public Sender decimalColumn(CharSequence name, Decimal256 value) { + if (value == null || value.isNull()) return this; checkNotClosed(); - // Fast path: if table name matches current, skip hashmap lookup - if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { - return this; - } - // Table changed - invalidate cached column references - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - currentTableName = tableName.toString(); - currentTableBuffer = tableBuffers.get(currentTableName); - if (currentTableBuffer == null) { - currentTableBuffer = new QwpTableBuffer(currentTableName); - tableBuffers.put(currentTableName, currentTableBuffer); - } - // Both modes accumulate rows until flush + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(value); return this; } @Override - public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { + public Sender decimalColumn(CharSequence name, CharSequence value) { + if (value == null || value.length() == 0) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); - - if (value != null) { - // Register symbol in global dictionary and track max ID for delta calculation - String symbolValue = value.toString(); - int globalId = globalSymbolDictionary.getOrAddSymbol(symbolValue); - if (globalId > currentBatchMaxSymbolId) { - currentBatchMaxSymbolId = globalId; - } - // Store global ID in the column buffer - col.addSymbolWithGlobalId(symbolValue, globalId); - } else { - col.addSymbol(null); + try { + java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); + Decimal256 decimal = Decimal256.fromBigDecimal(bd); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(decimal); + } catch (Exception e) { + throw new LineSenderException("Failed to parse decimal value: " + value, e); } return this; } @Override - public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { + public Sender doubleArray(@NotNull CharSequence name, double[] values) { + if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); - col.addBoolean(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); return this; } + // ==================== Sender interface implementation ==================== + @Override - public QwpWebSocketSender longColumn(CharSequence columnName, long value) { + public Sender doubleArray(@NotNull CharSequence name, double[][] values) { + if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); - col.addLong(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); return this; } - /** - * Adds an INT column value to the current row. - * - * @param columnName the column name - * @param value the int value - * @return this sender for method chaining - */ - /** - * Adds a BYTE column value to the current row. - * - * @param columnName the column name - * @param value the byte value - * @return this sender for method chaining - */ - public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { + if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false); - col.addByte(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); return this; } - public QwpWebSocketSender intColumn(CharSequence columnName, int value) { + @Override + public Sender doubleArray(CharSequence name, DoubleArray array) { + if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); - col.addInt(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(array); return this; } @@ -628,6 +594,14 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { return this; } + /** + * Adds an INT column value to the current row. + * + * @param columnName the column name + * @param value the int value + * @return this sender for method chaining + */ + /** * Adds a FLOAT column value to the current row. * @@ -644,62 +618,155 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { } @Override - public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + public void flush() { checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); - col.addString(value != null ? value.toString() : null); - return this; - } + ensureConnected(); - /** - * Adds a SHORT column value to the current row. - * - * @param columnName the column name - * @param value the short value - * @return this sender for method chaining - */ - public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); - col.addShort(value); - return this; - } + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush pending rows and wait for ACKs + flushPendingRows(); - /** - * Adds a CHAR column value to the current row. - *

- * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB. - * - * @param columnName the column name - * @param value the character value - * @return this sender for method chaining + // Flush any remaining data in the active microbatch buffer + if (activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + + // Wait for all pending batches to be sent to the server + sendQueue.flush(); + + // Wait for all in-flight batches to be acknowledged by the server + inFlightWindow.awaitEmpty(); + + LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); + } else { + // Sync mode (window=1): flush pending rows and wait for ACKs synchronously + flushSync(); + } + } + + /** + * Returns the auto-flush byte threshold. */ - public QwpWebSocketSender charColumn(CharSequence columnName, char value) { - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); - col.addShort((short) value); - return this; + public int getAutoFlushBytes() { + return autoFlushBytes; } /** - * Adds a UUID column value to the current row. - * - * @param columnName the column name - * @param lo the low 64 bits of the UUID - * @param hi the high 64 bits of the UUID - * @return this sender for method chaining + * Returns the auto-flush interval in nanoseconds. */ - public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { + public long getAutoFlushIntervalNanos() { + return autoFlushIntervalNanos; + } + + /** + * Returns the auto-flush row threshold. + */ + public int getAutoFlushRows() { + return autoFlushRows; + } + + /** + * Returns the global symbol dictionary. + * For testing and encoder integration. + */ + public GlobalSymbolDictionary getGlobalSymbolDictionary() { + return globalSymbolDictionary; + } + + /** + * Returns the in-flight window size. + * Window=1 means sync mode, window>1 means async mode. + */ + public int getInFlightWindowSize() { + return inFlightWindowSize; + } + + /** + * Returns the max symbol ID sent to the server. + * Once sent over TCP, server is guaranteed to receive it (or connection dies). + */ + public int getMaxSentSymbolId() { + return maxSentSymbolId; + } + + /** + * Registers a symbol in the global dictionary and returns its ID. + * For use with fast-path column buffer access. + */ + public int getOrAddGlobalSymbol(String value) { + int globalId = globalSymbolDictionary.getOrAddSymbol(value); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + return globalId; + } + + /** + * Returns the send queue capacity. + */ + public int getSendQueueCapacity() { + return sendQueueCapacity; + } + + /** + * Gets or creates a table buffer for direct access. + * For high-throughput generators that want to bypass fluent API overhead. + */ + public QwpTableBuffer getTableBuffer(String tableName) { + QwpTableBuffer buffer = tableBuffers.get(tableName); + if (buffer == null) { + buffer = new QwpTableBuffer(tableName); + tableBuffers.put(tableName, buffer); + } + currentTableBuffer = buffer; + currentTableName = tableName; + return buffer; + } + + /** + * Increments the pending row count for auto-flush tracking. + * Call this after adding a complete row via fast-path API. + * Triggers auto-flush if any threshold is exceeded. + */ + public void incrementPendingRowCount() { + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; + + // Check if any flush threshold is exceeded (same as sendRow()) + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); + } + } + } + + public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); - col.addUuid(hi, lo); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); + col.addInt(value); return this; } + /** + * Returns whether async mode is enabled (window size > 1). + */ + public boolean isAsyncMode() { + return inFlightWindowSize > 1; + } + + /** + * Returns whether Gorilla encoding is enabled. + */ + public boolean isGorillaEnabled() { + return gorillaEnabled; + } + /** * Adds a LONG256 column value to the current row. * @@ -718,6 +785,137 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l return this; } + @Override + public Sender longArray(@NotNull CharSequence name, long[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, LongArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(array); + return this; + } + + @Override + public QwpWebSocketSender longColumn(CharSequence columnName, long value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + col.addLong(value); + return this; + } + + @Override + public void reset() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.reset(); + } + } + + /** + * Sets whether to use Gorilla timestamp encoding. + */ + public QwpWebSocketSender setGorillaEnabled(boolean enabled) { + this.gorillaEnabled = enabled; + this.encoder.setGorillaEnabled(enabled); + return this; + } + + /** + * Adds a SHORT column value to the current row. + * + * @param columnName the column name + * @param value the short value + * @return this sender for method chaining + */ + public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); + col.addShort(value); + return this; + } + + @Override + public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + col.addString(value != null ? value.toString() : null); + return this; + } + + @Override + public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + + if (value != null) { + // Register symbol in global dictionary and track max ID for delta calculation + String symbolValue = value.toString(); + int globalId = globalSymbolDictionary.getOrAddSymbol(symbolValue); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + // Store global ID in the column buffer + col.addSymbolWithGlobalId(symbolValue, globalId); + } else { + col.addSymbol(null); + } + return this; + } + + @Override + public QwpWebSocketSender table(CharSequence tableName) { + checkNotClosed(); + // Fast path: if table name matches current, skip hashmap lookup + if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { + return this; + } + // Table changed - invalidate cached column references + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableName = tableName.toString(); + currentTableBuffer = tableBuffers.get(currentTableName); + if (currentTableBuffer == null) { + currentTableBuffer = new QwpTableBuffer(currentTableName); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Both modes accumulate rows until flush + return this; + } + @Override public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { checkNotClosed(); @@ -743,24 +941,44 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value return this; } - @Override - public void at(long timestamp, ChronoUnit unit) { + // ==================== Array methods ==================== + + /** + * Adds a UUID column value to the current row. + * + * @param columnName the column name + * @param lo the low 64 bits of the UUID + * @param hi the high 64 bits of the UUID + * @return this sender for method chaining + */ + public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { checkNotClosed(); checkTableSelected(); - if (unit == ChronoUnit.NANOS) { - atNanos(timestamp); - } else { - long micros = toMicros(timestamp, unit); - atMicros(micros); - } + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); + col.addUuid(hi, lo); + return this; } - @Override - public void at(Instant timestamp) { - checkNotClosed(); - checkTableSelected(); - long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; - atMicros(micros); + /** + * Adds encoded data to the active microbatch buffer. + * Triggers seal and swap if buffer is full. + */ + private void addToMicrobatch(long dataPtr, int length) { + // Ensure activeBuffer is ready for writing + ensureActiveBufferReady(); + + // If current buffer can't hold the data, seal and swap + if (activeBuffer.hasData() && + activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { + sealAndSwapBuffer(); + } + + // Ensure buffer can hold the data + activeBuffer.ensureCapacity(activeBuffer.getBufferPos() + length); + + // Copy data to buffer + activeBuffer.write(dataPtr, length); + activeBuffer.incrementRowCount(); } private void atMicros(long timestampMicros) { @@ -783,60 +1001,98 @@ private void atNanos(long timestampNanos) { sendRow(); } - @Override - public void atNow() { - checkNotClosed(); - checkTableSelected(); - // Server-assigned timestamp - just send the row without designated timestamp - sendRow(); + private void checkNotClosed() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + } + + private void checkTableSelected() { + if (currentTableBuffer == null) { + throw new LineSenderException("table() must be called before adding columns"); + } } /** - * Accumulates the current row. - * Both sync and async modes buffer rows until flush (explicit or auto-flush). - * The difference is that sync mode flush() blocks until server ACKs. + * Ensures the active buffer is ready for writing (in FILLING state). + * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. */ - private void sendRow() { - ensureConnected(); - currentTableBuffer.nextRow(); + private void ensureActiveBufferReady() { + if (activeBuffer.isFilling()) { + return; // Already ready + } - // Both modes: accumulate rows, don't encode yet - if (pendingRowCount == 0) { - firstPendingRowTimeNanos = System.nanoTime(); + if (activeBuffer.isRecycled()) { + // Buffer was recycled but not reset - reset it now + activeBuffer.reset(); + return; } - pendingRowCount++; - // Check if any flush threshold is exceeded - if (shouldAutoFlush()) { - if (inFlightWindowSize > 1) { - flushPendingRows(); - } else { - // Sync mode (window=1): flush directly with ACK wait - flushSync(); + // Buffer is in use (SEALED or SENDING) - wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for active buffer to be recycled"); } } - } - /** - * Checks if any auto-flush threshold is exceeded. - */ - private boolean shouldAutoFlush() { - if (pendingRowCount <= 0) { - return false; + // Buffer should now be RECYCLED - reset it + if (activeBuffer.isRecycled()) { + activeBuffer.reset(); } - // Row limit - if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { - return true; + } + + private void ensureConnected() { + if (closed) { + throw new LineSenderException("Sender is closed"); } - // Time limit - if (autoFlushIntervalNanos > 0) { - long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; - if (ageNanos >= autoFlushIntervalNanos) { - return true; + if (!connected) { + // Create WebSocket client using factory (zero-GC native implementation) + if (tlsEnabled) { + client = WebSocketClientFactory.newInsecureTlsInstance(); + } else { + client = WebSocketClientFactory.newPlainTextInstance(); + } + + // Connect and upgrade to WebSocket + try { + client.connect(host, port); + client.upgrade(WRITE_PATH); + } catch (Exception e) { + client.close(); + client = null; + throw new LineSenderException("Failed to connect to " + host + ":" + port, e); + } + + // a window for tracking batches awaiting ACK (both modes) + inFlightWindow = new InFlightWindow(inFlightWindowSize, InFlightWindow.DEFAULT_TIMEOUT_MS); + + // Initialize send queue for async mode (window > 1) + // The send queue handles both sending AND receiving (single I/O thread) + if (inFlightWindowSize > 1) { + sendQueue = new WebSocketSendQueue(client, inFlightWindow, + sendQueueCapacity, + WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, + WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); } + // Sync mode (window=1): no send queue - we send and read ACKs synchronously + + // Clear sent schema hashes - server starts fresh on each connection + sentSchemaHashes.clear(); + + connected = true; + LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}]", host, port, inFlightWindowSize); + } + } + + // ==================== Decimal methods ==================== + + private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { + if (inFlightWindow != null && inFlightWindow.getLastError() == null) { + inFlightWindow.fail(expectedSequence, error); } - // Byte limit is harder to estimate without encoding, skip for now - return false; } /** @@ -916,56 +1172,79 @@ private void flushPendingRows() { } /** - * Ensures the active buffer is ready for writing (in FILLING state). - * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. + * Flushes pending rows synchronously, blocking until server ACKs. + * Used in sync mode for simpler, blocking operation. */ - private void ensureActiveBufferReady() { - if (activeBuffer.isFilling()) { - return; // Already ready - } - - if (activeBuffer.isRecycled()) { - // Buffer was recycled but not reset - reset it now - activeBuffer.reset(); + private void flushSync() { + if (pendingRowCount <= 0) { return; } - // Buffer is in use (SEALED or SENDING) - wait for it - // Use a while loop to handle spurious wakeups and race conditions with the latch - while (activeBuffer.isInUse()) { - LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); - boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); - if (!recycled) { - throw new LineSenderException("Timeout waiting for active buffer to be recycled"); + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); + + // Encode all table buffers that have data into a single message + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; + } + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; } - } - // Buffer should now be RECYCLED - reset it - if (activeBuffer.isRecycled()) { - activeBuffer.reset(); - } - } + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); - /** - * Adds encoded data to the active microbatch buffer. - * Triggers seal and swap if buffer is full. - */ - private void addToMicrobatch(long dataPtr, int length) { - // Ensure activeBuffer is ready for writing - ensureActiveBufferReady(); + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); - // If current buffer can't hold the data, seal and swap - if (activeBuffer.hasData() && - activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { - sealAndSwapBuffer(); + // Track schema key if this was the first time sending this schema + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + + if (messageSize > 0) { + QwpBufferWriter buffer = encoder.getBuffer(); + + // Track batch in InFlightWindow before sending + long batchSequence = nextBatchSequence++; + inFlightWindow.addInFlight(batchSequence); + + // Update maxSentSymbolId - once sent over TCP, server will receive it + maxSentSymbolId = currentBatchMaxSymbolId; + + LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef); + + // Send over WebSocket + client.sendBinary(buffer.getBufferPtr(), messageSize); + + // Wait for ACK synchronously + waitForAck(batchSequence); + } + + // Reset table buffer after sending + tableBuffer.reset(); + + // Reset batch-level symbol tracking + currentBatchMaxSymbolId = -1; } - // Ensure buffer can hold the data - activeBuffer.ensureCapacity(activeBuffer.getBufferPos() + length); + // Reset pending row tracking + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; - // Copy data to buffer - activeBuffer.write(dataPtr, length); - activeBuffer.incrementRowCount(); + LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); } /** @@ -1016,107 +1295,75 @@ private void sealAndSwapBuffer() { } } - @Override - public void flush() { - checkNotClosed(); + // ==================== Helper methods ==================== + + /** + * Accumulates the current row. + * Both sync and async modes buffer rows until flush (explicit or auto-flush). + * The difference is that sync mode flush() blocks until server ACKs. + */ + private void sendRow() { ensureConnected(); + currentTableBuffer.nextRow(); - if (inFlightWindowSize > 1) { - // Async mode (window > 1): flush pending rows and wait for ACKs - flushPendingRows(); + // Both modes: accumulate rows, don't encode yet + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; - // Flush any remaining data in the active microbatch buffer - if (activeBuffer.hasData()) { - sealAndSwapBuffer(); + // Check if any flush threshold is exceeded + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); } - - // Wait for all pending batches to be sent to the server - sendQueue.flush(); - - // Wait for all in-flight batches to be acknowledged by the server - inFlightWindow.awaitEmpty(); - - LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); - } else { - // Sync mode (window=1): flush pending rows and wait for ACKs synchronously - flushSync(); } } /** - * Flushes pending rows synchronously, blocking until server ACKs. - * Used in sync mode for simpler, blocking operation. + * Checks if any auto-flush threshold is exceeded. */ - private void flushSync() { + private boolean shouldAutoFlush() { if (pendingRowCount <= 0) { - return; + return false; } - - LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); - - // Encode all table buffers that have data into a single message - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } - - // Check if this schema has been sent before (use schema reference mode if so) - // Combined key includes table name since server caches by (tableName, schemaHash) - long schemaHash = tableBuffer.getSchemaHash(); - long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); - boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); - - // Encode this table's rows with delta symbol dictionary - int messageSize = encoder.encodeWithDeltaDict( - tableBuffer, - globalSymbolDictionary, - maxSentSymbolId, - currentBatchMaxSymbolId, - useSchemaRef - ); - - // Track schema key if this was the first time sending this schema - if (!useSchemaRef) { - sentSchemaHashes.add(schemaKey); - } - - if (messageSize > 0) { - QwpBufferWriter buffer = encoder.getBuffer(); - - // Track batch in InFlightWindow before sending - long batchSequence = nextBatchSequence++; - inFlightWindow.addInFlight(batchSequence); - - // Update maxSentSymbolId - once sent over TCP, server will receive it - maxSentSymbolId = currentBatchMaxSymbolId; - - LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef); - - // Send over WebSocket - client.sendBinary(buffer.getBufferPtr(), messageSize); - - // Wait for ACK synchronously - waitForAck(batchSequence); + // Row limit + if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { + return true; + } + // Time limit + if (autoFlushIntervalNanos > 0) { + long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; + if (ageNanos >= autoFlushIntervalNanos) { + return true; } + } + // Byte limit is harder to estimate without encoding, skip for now + return false; + } - // Reset table buffer after sending - tableBuffer.reset(); - - // Reset batch-level symbol tracking - currentBatchMaxSymbolId = -1; + private long toMicros(long value, ChronoUnit unit) { + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); } - - // Reset pending row tracking - pendingRowCount = 0; - firstPendingRowTimeNanos = 0; - - LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); } /** @@ -1187,253 +1434,4 @@ public void onClose(int code, String reason) { failExpectedIfNeeded(expectedSequence, timeout); throw timeout; } - - private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { - if (inFlightWindow != null && inFlightWindow.getLastError() == null) { - inFlightWindow.fail(expectedSequence, error); - } - } - - @Override - public DirectByteSlice bufferView() { - throw new LineSenderException("bufferView() is not supported for WebSocket sender"); - } - - @Override - public void cancelRow() { - checkNotClosed(); - if (currentTableBuffer != null) { - currentTableBuffer.cancelCurrentRow(); - } - } - - @Override - public void reset() { - checkNotClosed(); - if (currentTableBuffer != null) { - currentTableBuffer.reset(); - } - } - - // ==================== Array methods ==================== - - @Override - public Sender doubleArray(@NotNull CharSequence name, double[] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - return this; - } - - @Override - public Sender doubleArray(@NotNull CharSequence name, double[][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - return this; - } - - @Override - public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - return this; - } - - @Override - public Sender doubleArray(CharSequence name, DoubleArray array) { - if (array == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(array); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, long[] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(values); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, long[][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(values); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, long[][][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(values); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, LongArray array) { - if (array == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(array); - return this; - } - - // ==================== Decimal methods ==================== - - @Override - public Sender decimalColumn(CharSequence name, Decimal64 value) { - if (value == null || value.isNull()) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); - col.addDecimal64(value); - return this; - } - - @Override - public Sender decimalColumn(CharSequence name, Decimal128 value) { - if (value == null || value.isNull()) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); - col.addDecimal128(value); - return this; - } - - @Override - public Sender decimalColumn(CharSequence name, Decimal256 value) { - if (value == null || value.isNull()) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); - col.addDecimal256(value); - return this; - } - - @Override - public Sender decimalColumn(CharSequence name, CharSequence value) { - if (value == null || value.length() == 0) return this; - checkNotClosed(); - checkTableSelected(); - try { - java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); - Decimal256 decimal = Decimal256.fromBigDecimal(bd); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); - col.addDecimal256(decimal); - } catch (Exception e) { - throw new LineSenderException("Failed to parse decimal value: " + value, e); - } - return this; - } - - // ==================== Helper methods ==================== - - private long toMicros(long value, ChronoUnit unit) { - switch (unit) { - case NANOS: - return value / 1000L; - case MICROS: - return value; - case MILLIS: - return value * 1000L; - case SECONDS: - return value * 1_000_000L; - case MINUTES: - return value * 60_000_000L; - case HOURS: - return value * 3_600_000_000L; - case DAYS: - return value * 86_400_000_000L; - default: - throw new LineSenderException("Unsupported time unit: " + unit); - } - } - - private void checkNotClosed() { - if (closed) { - throw new LineSenderException("Sender is closed"); - } - } - - private void checkTableSelected() { - if (currentTableBuffer == null) { - throw new LineSenderException("table() must be called before adding columns"); - } - } - - @Override - public void close() { - if (!closed) { - closed = true; - - // Flush any remaining data - try { - if (inFlightWindowSize > 1) { - // Async mode (window > 1): flush accumulated rows in table buffers first - flushPendingRows(); - - if (activeBuffer != null && activeBuffer.hasData()) { - sealAndSwapBuffer(); - } - if (sendQueue != null) { - sendQueue.close(); - } - } else { - // Sync mode (window=1): flush pending rows synchronously - if (pendingRowCount > 0 && client != null && client.isConnected()) { - flushSync(); - } - } - } catch (Exception e) { - LOG.error("Error during close: {}", String.valueOf(e)); - } - - // Close buffers (async mode only, window > 1) - if (buffer0 != null) { - buffer0.close(); - } - if (buffer1 != null) { - buffer1.close(); - } - - if (client != null) { - client.close(); - client = null; - } - encoder.close(); - // Close all table buffers to free off-heap column memory - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence key = keys.getQuick(i); - if (key != null) { - QwpTableBuffer tb = tableBuffers.get(key); - if (tb != null) { - tb.close(); - } - } - } - tableBuffers.clear(); - - LOG.info("QwpWebSocketSender closed"); - } - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 5281dbc..8f59b38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -143,7 +143,7 @@ public void encodeDoD(long deltaOfDelta) { } /** - * Encodes an array of timestamps to native memory using Gorilla compression. + * Encodes timestamps from off-heap memory using Gorilla compression. *

* Format: * 

@@ -157,11 +157,11 @@ public void encodeDoD(long deltaOfDelta) {
      *
      * @param destAddress destination address in native memory
      * @param capacity    maximum number of bytes to write
-     * @param timestamps  array of timestamp values
+     * @param srcAddress  source address of contiguous int64 timestamps in native memory
      * @param count       number of timestamps to encode
      * @return number of bytes written
      */
-    public int encodeTimestamps(long destAddress, long capacity, long[] timestamps, int count) {
+    public int encodeTimestamps(long destAddress, long capacity, long srcAddress, int count) {
         if (count == 0) {
             return 0;
         }
@@ -172,7 +172,8 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps,
         if (capacity < 8) {
             return 0; // Not enough space
         }
-        Unsafe.getUnsafe().putLong(destAddress, timestamps[0]);
+        long ts0 = Unsafe.getUnsafe().getLong(srcAddress);
+        Unsafe.getUnsafe().putLong(destAddress, ts0);
         pos = 8;
 
         if (count == 1) {
@@ -183,7 +184,8 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps,
         if (capacity < pos + 8) {
             return pos; // Not enough space
         }
-        Unsafe.getUnsafe().putLong(destAddress + pos, timestamps[1]);
+        long ts1 = Unsafe.getUnsafe().getLong(srcAddress + 8);
+        Unsafe.getUnsafe().putLong(destAddress + pos, ts1);
         pos += 8;
 
         if (count == 2) {
@@ -192,39 +194,41 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps,
 
         // Encode remaining with delta-of-delta
         bitWriter.reset(destAddress + pos, capacity - pos);
-        long prevTs = timestamps[1];
-        long prevDelta = timestamps[1] - timestamps[0];
+        long prevTs = ts1;
+        long prevDelta = ts1 - ts0;
 
         for (int i = 2; i < count; i++) {
-            long delta = timestamps[i] - prevTs;
+            long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8);
+            long delta = ts - prevTs;
             long dod = delta - prevDelta;
             encodeDoD(dod);
             prevDelta = delta;
-            prevTs = timestamps[i];
+            prevTs = ts;
         }
 
         return pos + bitWriter.finish();
     }
 
     /**
-     * Checks if Gorilla encoding can be used for the given timestamps.
+     * Checks if Gorilla encoding can be used for timestamps stored off-heap.
      * 

      * Gorilla encoding uses 32-bit signed integers for delta-of-delta values,
      * so it cannot encode timestamps where the delta-of-delta exceeds the
      * 32-bit signed integer range.
      *
-     * @param timestamps array of timestamp values
+     * @param srcAddress source address of contiguous int64 timestamps in native memory
      * @param count      number of timestamps
      * @return true if Gorilla encoding can be used, false otherwise
      */
-    public static boolean canUseGorilla(long[] timestamps, int count) {
+    public static boolean canUseGorilla(long srcAddress, int count) {
         if (count < 3) {
             return true; // No DoD encoding needed for 0, 1, or 2 timestamps
         }
 
-        long prevDelta = timestamps[1] - timestamps[0];
+        long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress);
         for (int i = 2; i < count; i++) {
-            long delta = timestamps[i] - timestamps[i - 1];
+            long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8)
+                    - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8);
             long dod = delta - prevDelta;
             if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) {
                 return false;
@@ -235,16 +239,16 @@ public static boolean canUseGorilla(long[] timestamps, int count) {
     }
 
     /**
-     * Calculates the encoded size in bytes for Gorilla-encoded timestamps.
+     * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap.
      * 

      * Note: This does NOT include the encoding flag byte. Add 1 byte if
      * the encoding flag is needed.
      *
-     * @param timestamps array of timestamp values
+     * @param srcAddress source address of contiguous int64 timestamps in native memory
      * @param count      number of timestamps
      * @return encoded size in bytes (excluding encoding flag)
      */
-    public static int calculateEncodedSize(long[] timestamps, int count) {
+    public static int calculateEncodedSize(long srcAddress, int count) {
         if (count == 0) {
             return 0;
         }
@@ -262,18 +266,19 @@ public static int calculateEncodedSize(long[] timestamps, int count) {
         }
 
         // Calculate bits for delta-of-delta encoding
-        long prevTimestamp = timestamps[1];
-        long prevDelta = timestamps[1] - timestamps[0];
+        long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8);
+        long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress);
         int totalBits = 0;
 
         for (int i = 2; i < count; i++) {
-            long delta = timestamps[i] - prevTimestamp;
+            long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8);
+            long delta = ts - prevTimestamp;
             long deltaOfDelta = delta - prevDelta;
 
             totalBits += getBitsRequired(deltaOfDelta);
 
             prevDelta = delta;
-            prevTimestamp = timestamps[i];
+            prevTimestamp = ts;
         }
 
         // Round up to bytes

From 87950812067c6b26ad3f0ab87f1c04f77c8bdb43 Mon Sep 17 00:00:00 2001
From: Jaromir Hamala 
Date: Mon, 23 Feb 2026 10:30:34 +0100
Subject: [PATCH 015/230] move test workloads/benchmarks to the client maven
 module

---
 .../line/tcp/v4/QwpAllocationTestClient.java  | 379 ++++++++++++++++
 .../line/tcp/v4/StacBenchmarkClient.java      | 424 ++++++++++++++++++
 2 files changed, 803 insertions(+)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
new file mode 100644
index 0000000..5eab4bf
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
@@ -0,0 +1,379 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2024 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.line.tcp.v4;
+
+import io.questdb.client.Sender;
+
+import java.io.IOException;
+import java.time.temporal.ChronoUnit;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test client for ILP allocation profiling.
+ * 

+ * Supports 3 protocol modes:
+ * 
    
+ *   
  • ilp-tcp: Old ILP text protocol over TCP (port 9009)
    • 
+ *   
  • ilp-http: Old ILP text protocol over HTTP (port 9000)
    • 
+ *   
  • qwp-websocket: New QWP binary protocol over WebSocket (port 9000)
    • 
+ * 

+ * 

+ * Sends rows with various column types to exercise all code paths.
+ * Run with an allocation profiler (async-profiler, JFR, etc.) to find hotspots.
+ * 

+ * Usage:
+ * 
+ * java -cp ... QwpAllocationTestClient [options]
+ *
+ * Options:
+ *   --protocol=PROTOCOL   Protocol: ilp-tcp, ilp-http, qwp-websocket (default: qwp-websocket)
+ *   --host=HOST           Server host (default: localhost)
+ *   --port=PORT           Server port (default: 9009 for TCP, 9000 for HTTP)
+ *   --rows=N              Total rows to send (default: 10000000)
+ *   --batch=N             Batch/flush size (default: 10000)
+ *   --warmup=N            Warmup rows (default: 100000)
+ *   --report=N            Report progress every N rows (default: 1000000)
+ *   --no-warmup           Skip warmup phase
+ *   --help                Show this help
+ *
+ * Examples:
+ *   QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000
+ *   QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server --port=9009
+ * 

+ */
+public class QwpAllocationTestClient {
+
+    // Protocol modes
+    private static final String PROTOCOL_ILP_TCP = "ilp-tcp";
+    private static final String PROTOCOL_ILP_HTTP = "ilp-http";
+    private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket";
+
+
+    // Default configuration
+    private static final String DEFAULT_HOST = "localhost";
+    private static final int DEFAULT_ROWS = 80_000_000;
+    private static final int DEFAULT_BATCH_SIZE = 10_000;
+    private static final int DEFAULT_FLUSH_BYTES = 0; // 0 = use protocol default
+    private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; // 0 = use protocol default
+    private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8)
+    private static final int DEFAULT_SEND_QUEUE = 0; // 0 = use protocol default (16)
+    private static final int DEFAULT_WARMUP_ROWS = 100_000;
+    private static final int DEFAULT_REPORT_INTERVAL = 1_000_000;
+
+    // Pre-computed test data to avoid allocation during the test
+    private static final String[] SYMBOLS = {
+            "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ",
+            "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA"
+    };
+
+    private static final String[] STRINGS = {
+            "New York", "London", "Tokyo", "Paris", "Berlin", "Sydney", "Toronto", "Singapore",
+            "Hong Kong", "Dubai", "Mumbai", "Shanghai", "Moscow", "Seoul", "Bangkok",
+            "Amsterdam", "Zurich", "Frankfurt", "Milan", "Madrid"
+    };
+
+    public static void main(String[] args) {
+        // Parse command-line options
+        String protocol = PROTOCOL_QWP_WEBSOCKET;
+        String host = DEFAULT_HOST;
+        int port = -1; // -1 means use default for protocol
+        int totalRows = DEFAULT_ROWS;
+        int batchSize = DEFAULT_BATCH_SIZE;
+        int flushBytes = DEFAULT_FLUSH_BYTES;
+        long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
+        int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW;
+        int sendQueue = DEFAULT_SEND_QUEUE;
+        int warmupRows = DEFAULT_WARMUP_ROWS;
+        int reportInterval = DEFAULT_REPORT_INTERVAL;
+
+        for (String arg : args) {
+            if (arg.equals("--help") || arg.equals("-h")) {
+                printUsage();
+                System.exit(0);
+            } else if (arg.startsWith("--protocol=")) {
+                protocol = arg.substring("--protocol=".length()).toLowerCase();
+            } else if (arg.startsWith("--host=")) {
+                host = arg.substring("--host=".length());
+            } else if (arg.startsWith("--port=")) {
+                port = Integer.parseInt(arg.substring("--port=".length()));
+            } else if (arg.startsWith("--rows=")) {
+                totalRows = Integer.parseInt(arg.substring("--rows=".length()));
+            } else if (arg.startsWith("--batch=")) {
+                batchSize = Integer.parseInt(arg.substring("--batch=".length()));
+            } else if (arg.startsWith("--flush-bytes=")) {
+                flushBytes = Integer.parseInt(arg.substring("--flush-bytes=".length()));
+            } else if (arg.startsWith("--flush-interval-ms=")) {
+                flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length()));
+            } else if (arg.startsWith("--in-flight-window=")) {
+                inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length()));
+            } else if (arg.startsWith("--send-queue=")) {
+                sendQueue = Integer.parseInt(arg.substring("--send-queue=".length()));
+            } else if (arg.startsWith("--warmup=")) {
+                warmupRows = Integer.parseInt(arg.substring("--warmup=".length()));
+            } else if (arg.startsWith("--report=")) {
+                reportInterval = Integer.parseInt(arg.substring("--report=".length()));
+            } else if (arg.equals("--no-warmup")) {
+                warmupRows = 0;
+            } else if (!arg.startsWith("--")) {
+                // Legacy positional args: protocol [host] [port] [rows]
+                protocol = arg.toLowerCase();
+            } else {
+                System.err.println("Unknown option: " + arg);
+                printUsage();
+                System.exit(1);
+            }
+        }
+
+        // Use default port if not specified
+        if (port == -1) {
+            port = getDefaultPort(protocol);
+        }
+
+        System.out.println("ILP Allocation Test Client");
+        System.out.println("==========================");
+        System.out.println("Protocol: " + protocol);
+        System.out.println("Host: " + host);
+        System.out.println("Port: " + port);
+        System.out.println("Total rows: " + String.format("%,d", totalRows));
+        System.out.println("Batch size (rows): " + String.format("%,d", batchSize) + (batchSize == 0 ? " (default)" : ""));
+        System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes)));
+        System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms"));
+        System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow));
+        System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue));
+        System.out.println("Warmup rows: " + String.format("%,d", warmupRows));
+        System.out.println("Report interval: " + String.format("%,d", reportInterval));
+        System.out.println();
+
+        try {
+            runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs,
+                    inFlightWindow, sendQueue, warmupRows, reportInterval);
+        } catch (Exception e) {
+            System.err.println("Error: " + e.getMessage());
+            e.printStackTrace(System.err);
+            System.exit(1);
+        }
+    }
+
+    private static void printUsage() {
+        System.out.println("ILP Allocation Test Client");
+        System.out.println();
+        System.out.println("Usage: QwpAllocationTestClient [options]");
+        System.out.println();
+        System.out.println("Options:");
+        System.out.println("  --protocol=PROTOCOL      Protocol to use (default: qwp-websocket)");
+        System.out.println("  --host=HOST              Server host (default: localhost)");
+        System.out.println("  --port=PORT              Server port (default: 9009 for TCP, 9000 for HTTP/WebSocket)");
+        System.out.println("  --rows=N                 Total rows to send (default: 80000000)");
+        System.out.println("  --batch=N                Auto-flush after N rows (default: 10000)");
+        System.out.println("  --flush-bytes=N          Auto-flush after N bytes (default: protocol default)");
+        System.out.println("  --flush-interval-ms=N    Auto-flush after N ms (default: protocol default)");
+        System.out.println("  --in-flight-window=N     Max batches awaiting server ACK (default: 8, WebSocket only)");
+        System.out.println("  --send-queue=N           Max batches waiting to send (default: 16, WebSocket only)");
+        System.out.println("  --warmup=N               Warmup rows (default: 100000)");
+        System.out.println("  --report=N               Report progress every N rows (default: 1000000)");
+        System.out.println("  --no-warmup              Skip warmup phase");
+        System.out.println("  --help                   Show this help");
+        System.out.println();
+        System.out.println("Protocols:");
+        System.out.println("  ilp-tcp          Old ILP text protocol over TCP (default port: 9009)");
+        System.out.println("  ilp-http         Old ILP text protocol over HTTP (default port: 9000)");
+        System.out.println("  qwp-websocket    New QWP binary protocol over WebSocket (default port: 9000)");
+        System.out.println();
+        System.out.println("Examples:");
+        System.out.println("  QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000");
+        System.out.println("  QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server");
+        System.out.println("  QwpAllocationTestClient --protocol=ilp-tcp --rows=100000 --no-warmup");
+    }
+
+    private static int getDefaultPort(String protocol) {
+        if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) {
+            return 9000;
+        }
+        return 9009;
+    }
+
+    private static void runTest(String protocol, String host, int port, int totalRows,
+                                  int batchSize, int flushBytes, long flushIntervalMs,
+                                  int inFlightWindow, int sendQueue,
+                                  int warmupRows, int reportInterval) throws IOException {
+        System.out.println("Connecting to " + host + ":" + port + "...");
+
+        try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs,
+                inFlightWindow, sendQueue)) {
+            System.out.println("Connected! Protocol: " + protocol);
+            System.out.println();
+
+            // Warm-up phase
+            if (warmupRows > 0) {
+                System.out.println("Warming up (" + String.format("%,d", warmupRows) + " rows)...");
+                long warmupStart = System.nanoTime();
+                for (int i = 0; i < warmupRows; i++) {
+                    sendRow(sender, i);
+                }
+                sender.flush();
+                long warmupTime = System.nanoTime() - warmupStart;
+                System.out.println("Warmup complete in " + TimeUnit.NANOSECONDS.toMillis(warmupTime) + " ms");
+                System.out.println();
+
+                // Give GC a chance to clean up warmup allocations
+                System.gc();
+                Thread.sleep(100);
+            }
+
+            // Main test phase
+            System.out.println("Starting main test (" + String.format("%,d", totalRows) + " rows)...");
+            if (reportInterval > 0 && reportInterval <= totalRows) {
+                System.out.println("Progress will be reported every " + String.format("%,d", reportInterval) + " rows");
+            }
+            System.out.println();
+
+            long startTime = System.nanoTime();
+            long lastReportTime = startTime;
+            int lastReportRows = 0;
+
+            for (int i = 0; i < totalRows; i++) {
+                sendRow(sender, i);
+
+                // Report progress
+                if (reportInterval > 0 && (i + 1) % reportInterval == 0) {
+                    long now = System.nanoTime();
+                    long elapsedSinceReport = now - lastReportTime;
+                    int rowsSinceReport = (i + 1) - lastReportRows;
+                    double rowsPerSec = rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0);
+
+                    System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %.0f rows/sec%n",
+                            i + 1, totalRows,
+                            (i + 1) * 100.0 / totalRows,
+                            rowsPerSec);
+
+                    lastReportTime = now;
+                    lastReportRows = i + 1;
+                }
+            }
+
+            // Final flush
+            sender.flush();
+
+            long endTime = System.nanoTime();
+            long totalTime = endTime - startTime;
+            double totalSeconds = totalTime / 1_000_000_000.0;
+            double rowsPerSecond = totalRows / totalSeconds;
+
+            System.out.println();
+            System.out.println("Test Complete!");
+            System.out.println("==============");
+            System.out.println("Protocol: " + protocol);
+            System.out.println("Total rows: " + String.format("%,d", totalRows));
+            System.out.println("Batch size: " + String.format("%,d", batchSize));
+            System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds");
+            System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second");
+            System.out.println("Data rate (before compression): " + String.format("%.2f", ((long)totalRows * estimatedRowSize()) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)");
+
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted", e);
+        }
+    }
+
+    private static Sender createSender(String protocol, String host, int port,
+                                         int batchSize, int flushBytes, long flushIntervalMs,
+                                         int inFlightWindow, int sendQueue) {
+        switch (protocol) {
+            case PROTOCOL_ILP_TCP:
+                return Sender.builder(Sender.Transport.TCP)
+                        .address(host)
+                        .port(port)
+                        .build();
+            case PROTOCOL_ILP_HTTP:
+                return Sender.builder(Sender.Transport.HTTP)
+                        .address(host)
+                        .port(port)
+                        .autoFlushRows(batchSize)
+                        .build();
+            case PROTOCOL_QWP_WEBSOCKET:
+                Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET)
+                        .address(host)
+                        .port(port)
+                        .asyncMode(true);
+                if (batchSize > 0) b.autoFlushRows(batchSize);
+                if (flushBytes > 0) b.autoFlushBytes(flushBytes);
+                if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs);
+                if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow);
+                if (sendQueue > 0) b.sendQueueCapacity(sendQueue);
+                return b.build();
+            default:
+                throw new IllegalArgumentException("Unknown protocol: " + protocol +
+                        ". Use one of: ilp-tcp, ilp-http, qwp-websocket");
+        }
+    }
+
+    private static void sendRow(Sender sender, int rowIndex) {
+        // Base timestamp with small variations
+        long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros
+        long timestamp = baseTimestamp + (rowIndex * 1000L) + (rowIndex % 100);
+
+        sender.table("ilp_alloc_test")
+                // Symbol columns
+                .symbol("exchange", SYMBOLS[rowIndex % SYMBOLS.length])
+                .symbol("currency", rowIndex % 2 == 0 ? "USD" : "EUR")
+
+                // Numeric columns
+                .longColumn("trade_id", rowIndex)
+                .longColumn("volume", 100 + (rowIndex % 10000))
+                .doubleColumn("price", 100.0 + (rowIndex % 1000) * 0.01)
+                .doubleColumn("bid", 99.5 + (rowIndex % 1000) * 0.01)
+                .doubleColumn("ask", 100.5 + (rowIndex % 1000) * 0.01)
+                .longColumn("sequence", rowIndex % 1000000)
+                .doubleColumn("spread", 0.5 + (rowIndex % 100) * 0.01)
+
+                // String column
+                .stringColumn("venue", STRINGS[rowIndex % STRINGS.length])
+
+                // Boolean column
+                .boolColumn("is_buy", rowIndex % 2 == 0)
+
+                // Additional timestamp column
+                .timestampColumn("event_time", timestamp - 1000, ChronoUnit.MICROS)
+
+                // Designated timestamp
+                .at(timestamp, ChronoUnit.MICROS);
+    }
+
+    /**
+     * Estimates the size of a single row in bytes for throughput calculation.
+     */
+    private static int estimatedRowSize() {
+        // Rough estimate (binary protocol):
+        // - 2 symbols: ~10 bytes each = 20 bytes
+        // - 3 longs: 8 bytes each = 24 bytes
+        // - 4 doubles: 8 bytes each = 32 bytes
+        // - 1 string: ~10 bytes average
+        // - 1 boolean: 1 byte
+        // - 2 timestamps: 8 bytes each = 16 bytes
+        // - Overhead: ~20 bytes
+        // Total: ~123 bytes
+        return 123;
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java
new file mode 100644
index 0000000..abbf41f
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java
@@ -0,0 +1,424 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2024 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.line.tcp.v4;
+
+import io.questdb.client.Sender;
+
+import java.io.IOException;
+import java.time.temporal.ChronoUnit;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * STAC benchmark ingestion test client.
+ * 

+ * Tests ingestion performance for a STAC-like quotes table with this schema:
+ * 
+ * CREATE TABLE q (
+ *     s SYMBOL,     -- 4-letter ticker symbol (8512 unique)
+ *     x CHAR,       -- exchange code
+ *     b FLOAT,      -- bid price
+ *     a FLOAT,      -- ask price
+ *     v SHORT,      -- bid volume
+ *     w SHORT,      -- ask volume
+ *     m BOOLEAN,    -- market flag
+ *     T TIMESTAMP   -- designated timestamp
+ * ) timestamp(T) PARTITION BY DAY WAL;
+ * 

+ * 

+ * The table MUST be pre-created before running this test so the server uses
+ * the correct narrow column types (FLOAT, SHORT, CHAR). Otherwise ILP
+ * auto-creation would use DOUBLE, LONG, STRING.
+ * 

+ * Supports 3 protocol modes:
+ * 
    
+ *   
  • ilp-tcp: Old ILP text protocol over TCP (port 9009)
    • 
+ *   
  • ilp-http: Old ILP text protocol over HTTP (port 9000)
    • 
+ *   
  • qwp-websocket: New QWP binary protocol over WebSocket (port 9000)
    • 
+ * 

+ */
+public class StacBenchmarkClient {
+
+    private static final String PROTOCOL_ILP_TCP = "ilp-tcp";
+    private static final String PROTOCOL_ILP_HTTP = "ilp-http";
+    private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket";
+
+    private static final String DEFAULT_HOST = "localhost";
+    private static final int DEFAULT_ROWS = 80_000_000;
+    private static final int DEFAULT_BATCH_SIZE = 10_000;
+    private static final int DEFAULT_FLUSH_BYTES = 0;
+    private static final long DEFAULT_FLUSH_INTERVAL_MS = 0;
+    private static final int DEFAULT_IN_FLIGHT_WINDOW = 0;
+    private static final int DEFAULT_SEND_QUEUE = 0;
+    private static final int DEFAULT_WARMUP_ROWS = 100_000;
+    private static final int DEFAULT_REPORT_INTERVAL = 1_000_000;
+    private static final String DEFAULT_TABLE = "q";
+
+    // 8512 unique 4-letter symbols, as per STAC NYSE benchmark
+    private static final int SYMBOL_COUNT = 8512;
+    private static final String[] SYMBOLS = generateSymbols(SYMBOL_COUNT);
+
+    // Exchange codes (single characters)
+    private static final char[] EXCHANGES = {'N', 'Q', 'A', 'B', 'C', 'D', 'P', 'Z'};
+    // Pre-computed single-char strings to avoid allocation
+    private static final String[] EXCHANGE_STRINGS = new String[EXCHANGES.length];
+
+    static {
+        for (int i = 0; i < EXCHANGES.length; i++) {
+            EXCHANGE_STRINGS[i] = String.valueOf(EXCHANGES[i]);
+        }
+    }
+
+    // Pre-computed bid base prices per symbol (to generate realistic spreads)
+    private static final float[] BASE_PRICES = generateBasePrices(SYMBOL_COUNT);
+
+    public static void main(String[] args) {
+        String protocol = PROTOCOL_QWP_WEBSOCKET;
+        String host = DEFAULT_HOST;
+        int port = -1;
+        int totalRows = DEFAULT_ROWS;
+        int batchSize = DEFAULT_BATCH_SIZE;
+        int flushBytes = DEFAULT_FLUSH_BYTES;
+        long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
+        int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW;
+        int sendQueue = DEFAULT_SEND_QUEUE;
+        int warmupRows = DEFAULT_WARMUP_ROWS;
+        int reportInterval = DEFAULT_REPORT_INTERVAL;
+        String table = DEFAULT_TABLE;
+
+        for (String arg : args) {
+            if (arg.equals("--help") || arg.equals("-h")) {
+                printUsage();
+                System.exit(0);
+            } else if (arg.startsWith("--protocol=")) {
+                protocol = arg.substring("--protocol=".length()).toLowerCase();
+            } else if (arg.startsWith("--host=")) {
+                host = arg.substring("--host=".length());
+            } else if (arg.startsWith("--port=")) {
+                port = Integer.parseInt(arg.substring("--port=".length()));
+            } else if (arg.startsWith("--rows=")) {
+                totalRows = Integer.parseInt(arg.substring("--rows=".length()));
+            } else if (arg.startsWith("--batch=")) {
+                batchSize = Integer.parseInt(arg.substring("--batch=".length()));
+            } else if (arg.startsWith("--flush-bytes=")) {
+                flushBytes = Integer.parseInt(arg.substring("--flush-bytes=".length()));
+            } else if (arg.startsWith("--flush-interval-ms=")) {
+                flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length()));
+            } else if (arg.startsWith("--in-flight-window=")) {
+                inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length()));
+            } else if (arg.startsWith("--send-queue=")) {
+                sendQueue = Integer.parseInt(arg.substring("--send-queue=".length()));
+            } else if (arg.startsWith("--warmup=")) {
+                warmupRows = Integer.parseInt(arg.substring("--warmup=".length()));
+            } else if (arg.startsWith("--report=")) {
+                reportInterval = Integer.parseInt(arg.substring("--report=".length()));
+            } else if (arg.startsWith("--table=")) {
+                table = arg.substring("--table=".length());
+            } else if (arg.equals("--no-warmup")) {
+                warmupRows = 0;
+            } else {
+                System.err.println("Unknown option: " + arg);
+                printUsage();
+                System.exit(1);
+            }
+        }
+
+        if (port == -1) {
+            port = getDefaultPort(protocol);
+        }
+
+        System.out.println("STAC Benchmark Ingestion Client");
+        System.out.println("================================");
+        System.out.println("Protocol: " + protocol);
+        System.out.println("Host: " + host);
+        System.out.println("Port: " + port);
+        System.out.println("Table: " + table);
+        System.out.println("Total rows: " + String.format("%,d", totalRows));
+        System.out.println("Batch size (rows): " + String.format("%,d", batchSize) + (batchSize == 0 ? " (default)" : ""));
+        System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes)));
+        System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms"));
+        System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow));
+        System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue));
+        System.out.println("Warmup rows: " + String.format("%,d", warmupRows));
+        System.out.println("Report interval: " + String.format("%,d", reportInterval));
+        System.out.println("Symbols: " + String.format("%,d", SYMBOL_COUNT) + " unique 4-letter tickers");
+        System.out.println();
+
+        try {
+            runTest(protocol, host, port, table, totalRows, batchSize, flushBytes, flushIntervalMs,
+                    inFlightWindow, sendQueue, warmupRows, reportInterval);
+        } catch (Exception e) {
+            System.err.println("Error: " + e.getMessage());
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void printUsage() {
+        System.out.println("STAC Benchmark Ingestion Client");
+        System.out.println();
+        System.out.println("Tests ingestion performance for a STAC-like quotes table.");
+        System.out.println("The table must be pre-created with the correct schema.");
+        System.out.println();
+        System.out.println("Usage: StacBenchmarkClient [options]");
+        System.out.println();
+        System.out.println("Options:");
+        System.out.println("  --protocol=PROTOCOL      Protocol to use (default: qwp-websocket)");
+        System.out.println("  --host=HOST              Server host (default: localhost)");
+        System.out.println("  --port=PORT              Server port (default: 9009 for TCP, 9000 for HTTP/WebSocket)");
+        System.out.println("  --table=TABLE            Table name (default: q)");
+        System.out.println("  --rows=N                 Total rows to send (default: 80000000)");
+        System.out.println("  --batch=N                Auto-flush after N rows (default: 10000)");
+        System.out.println("  --flush-bytes=N          Auto-flush after N bytes (default: protocol default)");
+        System.out.println("  --flush-interval-ms=N    Auto-flush after N ms (default: protocol default)");
+        System.out.println("  --in-flight-window=N     Max batches awaiting server ACK (default: 8, WebSocket only)");
+        System.out.println("  --send-queue=N           Max batches waiting to send (default: 16, WebSocket only)");
+        System.out.println("  --warmup=N               Warmup rows (default: 100000)");
+        System.out.println("  --report=N               Report progress every N rows (default: 1000000)");
+        System.out.println("  --no-warmup              Skip warmup phase");
+        System.out.println("  --help                   Show this help");
+        System.out.println();
+        System.out.println("Protocols:");
+        System.out.println("  ilp-tcp          Old ILP text protocol over TCP (default port: 9009)");
+        System.out.println("  ilp-http         Old ILP text protocol over HTTP (default port: 9000)");
+        System.out.println("  qwp-websocket    New QWP binary protocol over WebSocket (default port: 9000)");
+        System.out.println();
+        System.out.println("Table schema (must be pre-created):");
+        System.out.println("  CREATE TABLE q (");
+        System.out.println("      s SYMBOL, x CHAR, b FLOAT, a FLOAT,");
+        System.out.println("      v SHORT, w SHORT, m BOOLEAN, T TIMESTAMP");
+        System.out.println("  ) timestamp(T) PARTITION BY DAY WAL;");
+    }
+
+    private static int getDefaultPort(String protocol) {
+        if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) {
+            return 9000;
+        }
+        return 9009;
+    }
+
+    private static void runTest(String protocol, String host, int port, String table,
+                                int totalRows, int batchSize, int flushBytes, long flushIntervalMs,
+                                int inFlightWindow, int sendQueue,
+                                int warmupRows, int reportInterval) throws IOException {
+        System.out.println("Connecting to " + host + ":" + port + "...");
+
+        try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs,
+                inFlightWindow, sendQueue)) {
+            System.out.println("Connected! Protocol: " + protocol);
+            System.out.println();
+
+            // Warmup phase
+            if (warmupRows > 0) {
+                System.out.println("Warming up (" + String.format("%,d", warmupRows) + " rows)...");
+                long warmupStart = System.nanoTime();
+                for (int i = 0; i < warmupRows; i++) {
+                    sendQuoteRow(sender, table, i);
+                }
+                sender.flush();
+                long warmupTime = System.nanoTime() - warmupStart;
+                double warmupRowsPerSec = warmupRows / (warmupTime / 1_000_000_000.0);
+                System.out.printf("Warmup complete in %d ms (%.0f rows/sec)%n",
+                        TimeUnit.NANOSECONDS.toMillis(warmupTime), warmupRowsPerSec);
+                System.out.println();
+
+                System.gc();
+                Thread.sleep(100);
+            }
+
+            // Main test phase
+            System.out.println("Starting main test (" + String.format("%,d", totalRows) + " rows)...");
+            if (reportInterval > 0 && reportInterval <= totalRows) {
+                System.out.println("Progress will be reported every " + String.format("%,d", reportInterval) + " rows");
+            }
+            System.out.println();
+
+            long startTime = System.nanoTime();
+            long lastReportTime = startTime;
+            int lastReportRows = 0;
+
+            for (int i = 0; i < totalRows; i++) {
+                sendQuoteRow(sender, table, i);
+
+                if (reportInterval > 0 && (i + 1) % reportInterval == 0) {
+                    long now = System.nanoTime();
+                    long elapsedSinceReport = now - lastReportTime;
+                    int rowsSinceReport = (i + 1) - lastReportRows;
+                    double rowsPerSec = rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0);
+                    long totalElapsed = now - startTime;
+                    double overallRowsPerSec = (i + 1) / (totalElapsed / 1_000_000_000.0);
+
+                    System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %.0f rows/sec (interval) - %.0f rows/sec (overall)%n",
+                            i + 1, totalRows,
+                            (i + 1) * 100.0 / totalRows,
+                            rowsPerSec, overallRowsPerSec);
+
+                    lastReportTime = now;
+                    lastReportRows = i + 1;
+                }
+            }
+
+            sender.flush();
+
+            long endTime = System.nanoTime();
+            long totalTime = endTime - startTime;
+            double totalSeconds = totalTime / 1_000_000_000.0;
+            double rowsPerSecond = totalRows / totalSeconds;
+
+            System.out.println();
+            System.out.println("Test Complete!");
+            System.out.println("==============");
+            System.out.println("Protocol: " + protocol);
+            System.out.println("Table: " + table);
+            System.out.println("Total rows: " + String.format("%,d", totalRows));
+            System.out.println("Batch size: " + String.format("%,d", batchSize));
+            System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds");
+            System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second");
+            System.out.println("Data rate (before compression): " + String.format("%.2f",
+                    ((long) totalRows * ESTIMATED_ROW_SIZE) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)");
+
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted", e);
+        }
+    }
+
+    private static Sender createSender(String protocol, String host, int port,
+                                       int batchSize, int flushBytes, long flushIntervalMs,
+                                       int inFlightWindow, int sendQueue) {
+        switch (protocol) {
+            case PROTOCOL_ILP_TCP:
+                return Sender.builder(Sender.Transport.TCP)
+                        .address(host)
+                        .port(port)
+                        .build();
+            case PROTOCOL_ILP_HTTP:
+                return Sender.builder(Sender.Transport.HTTP)
+                        .address(host)
+                        .port(port)
+                        .autoFlushRows(batchSize)
+                        .build();
+            case PROTOCOL_QWP_WEBSOCKET:
+                Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET)
+                        .address(host)
+                        .port(port)
+                        .asyncMode(true);
+                if (batchSize > 0) b.autoFlushRows(batchSize);
+                if (flushBytes > 0) b.autoFlushBytes(flushBytes);
+                if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs);
+                if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow);
+                if (sendQueue > 0) b.sendQueueCapacity(sendQueue);
+                return b.build();
+            default:
+                throw new IllegalArgumentException("Unknown protocol: " + protocol +
+                        ". Use one of: ilp-tcp, ilp-http, qwp-websocket");
+        }
+    }
+
+    /**
+     * Sends a single quote row matching the STAC schema.
+     * 

+     * Schema: s SYMBOL, x CHAR, b FLOAT, a FLOAT, v SHORT, w SHORT, m BOOLEAN, T TIMESTAMP
+     * 

+     * The server downcasts doubleColumn->FLOAT, longColumn->SHORT, stringColumn->CHAR
+     * when the table is pre-created with the correct schema.
+     */
+    private static void sendQuoteRow(Sender sender, String table, int rowIndex) {
+        int symbolIdx = rowIndex % SYMBOL_COUNT;
+        int exchangeIdx = rowIndex % EXCHANGES.length;
+
+        // Bid/ask prices: base price with small variation
+        float basePrice = BASE_PRICES[symbolIdx];
+        // Use rowIndex bits for fast pseudo-random variation without Random object
+        float variation = ((rowIndex * 7 + symbolIdx * 13) % 200 - 100) * 0.01f;
+        float bid = basePrice + variation;
+        float ask = bid + 0.01f + (rowIndex % 10) * 0.01f; // spread: 1-10 cents
+
+        // Volumes: 100-32000 range fits SHORT
+        short bidVol = (short) (100 + ((rowIndex * 3 + symbolIdx) % 31901));
+        short askVol = (short) (100 + ((rowIndex * 7 + symbolIdx * 5) % 31901));
+
+        // Timestamp: 1 day of data with microsecond precision
+        // 86,400,000,000 micros per day, spread across totalRows
+        long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros
+        long timestamp = baseTimestamp + (rowIndex * 10L) + (rowIndex % 7);
+
+        sender.table(table)
+                .symbol("s", SYMBOLS[symbolIdx])
+                .stringColumn("x", EXCHANGE_STRINGS[exchangeIdx])
+                .doubleColumn("b", bid)
+                .doubleColumn("a", ask)
+                .longColumn("v", bidVol)
+                .longColumn("w", askVol)
+                .boolColumn("m", (rowIndex & 1) == 0)
+                .at(timestamp, ChronoUnit.MICROS);
+    }
+
+    /**
+     * Generates N unique 4-letter symbols.
+     * Uses combinations of uppercase letters to produce predictable, reproducible symbols.
+     */
+    private static String[] generateSymbols(int count) {
+        String[] symbols = new String[count];
+        int idx = 0;
+        // 26^4 = 456,976 possible 4-letter combinations, far more than 8512
+        outer:
+        for (char a = 'A'; a <= 'Z' && idx < count; a++) {
+            for (char b = 'A'; b <= 'Z' && idx < count; b++) {
+                for (char c = 'A'; c <= 'Z' && idx < count; c++) {
+                    for (char d = 'A'; d <= 'Z' && idx < count; d++) {
+                        symbols[idx++] = new String(new char[]{a, b, c, d});
+                        if (idx >= count) break outer;
+                    }
+                }
+            }
+        }
+        return symbols;
+    }
+
+    /**
+     * Generates pseudo-random base prices for each symbol.
+     * Prices range from $1 to $500 to simulate realistic stock prices.
+     */
+    private static float[] generateBasePrices(int count) {
+        float[] prices = new float[count];
+        Random rng = new Random(42); // fixed seed for reproducibility
+        for (int i = 0; i < count; i++) {
+            prices[i] = 1.0f + rng.nextFloat() * 499.0f;
+        }
+        return prices;
+    }
+
+    // Estimated row size for throughput calculation:
+    // - 1 symbol: ~6 bytes (4-char + overhead)
+    // - 1 char: 2 bytes
+    // - 2 floats: 4 bytes each = 8 bytes
+    // - 2 shorts: 2 bytes each = 4 bytes
+    // - 1 boolean: 1 byte
+    // - 1 timestamp: 8 bytes
+    // - overhead: ~10 bytes
+    // Total: ~39 bytes
+    private static final int ESTIMATED_ROW_SIZE = 39;
+}
\ No newline at end of file

From 992cd8ed63da5fc24aa392e9b6767c1debeba431 Mon Sep 17 00:00:00 2001
From: bluestreak 
Date: Mon, 23 Feb 2026 21:47:41 +0000
Subject: [PATCH 016/230] wip 13

---
 .../qwp/client/QwpWebSocketEncoder.java       |  36 +--
 .../qwp/protocol/OffHeapAppendMemory.java     |  32 ++
 .../cutlass/qwp/protocol/QwpTableBuffer.java  | 298 +++++++++---------
 .../qwp/protocol/OffHeapAppendMemoryTest.java |  77 +++++
 4 files changed, 276 insertions(+), 167 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
index 8f8d2ac..0ec22b2 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.client;
 
+import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
 import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
@@ -204,7 +205,7 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef,
                 break;
             case TYPE_STRING:
             case TYPE_VARCHAR:
-                writeStringColumn(col.getStringValues(), valueCount);
+                writeStringColumn(col, valueCount);
                 break;
             case TYPE_SYMBOL:
                 writeSymbolColumn(col, valueCount);
@@ -233,7 +234,7 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef,
                 writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount);
                 break;
             default:
-                throw new IllegalStateException("Unknown column type: " + col.getType());
+                throw new LineSenderException("Unknown column type: " + col.getType());
         }
     }
 
@@ -281,7 +282,7 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC
                     break;
                 case TYPE_STRING:
                 case TYPE_VARCHAR:
-                    writeStringColumn(col.getStringValues(), valueCount);
+                    writeStringColumn(col, valueCount);
                     break;
                 case TYPE_UUID:
                     buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16);
@@ -305,7 +306,7 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC
                     writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount);
                     break;
                 default:
-                    throw new IllegalStateException("Unknown column type: " + col.getType());
+                    throw new LineSenderException("Unknown column type: " + col.getType());
             }
         }
     }
@@ -482,28 +483,11 @@ private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) {
         }
     }
 
-    private void writeStringColumn(String[] strings, int count) {
-        int totalDataLen = 0;
-        for (int i = 0; i < count; i++) {
-            if (strings[i] != null) {
-                totalDataLen += QwpBufferWriter.utf8Length(strings[i]);
-            }
-        }
-
-        int runningOffset = 0;
-        buffer.putInt(0);
-        for (int i = 0; i < count; i++) {
-            if (strings[i] != null) {
-                runningOffset += QwpBufferWriter.utf8Length(strings[i]);
-            }
-            buffer.putInt(runningOffset);
-        }
-
-        for (int i = 0; i < count; i++) {
-            if (strings[i] != null) {
-                buffer.putUtf8(strings[i]);
-            }
-        }
+    private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) {
+        // Offset array: (valueCount + 1) int32 values, pre-built in wire format
+        buffer.putBlockOfBytes(col.getStringOffsetsAddress(), (long) (valueCount + 1) * 4);
+        // UTF-8 data: raw bytes, contiguous
+        buffer.putBlockOfBytes(col.getStringDataAddress(), col.getStringDataSize());
     }
 
     /**
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
index f4c14cc..5830ea7 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
@@ -131,6 +131,38 @@ public void putDouble(double value) {
         appendAddress += 8;
     }
 
+    /**
+     * Encodes a Java String to UTF-8 directly into the off-heap buffer.
+     * Pre-ensures worst-case capacity to avoid per-byte checks.
+     */
+    public void putUtf8(String value) {
+        if (value == null || value.isEmpty()) {
+            return;
+        }
+        int len = value.length();
+        ensureCapacity((long) len * 4); // worst case: all supplementary chars
+        for (int i = 0; i < len; i++) {
+            char c = value.charAt(i);
+            if (c < 0x80) {
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) c);
+            } else if (c < 0x800) {
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xC0 | (c >> 6)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F)));
+            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < len) {
+                char c2 = value.charAt(++i);
+                int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00);
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F)));
+            } else {
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F)));
+            }
+        }
+    }
+
     /**
      * Advances the append position by the given number of bytes without writing.
      */
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index 9e97b4c..b2c434c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -52,16 +52,16 @@
  */
 public class QwpTableBuffer implements QuietCloseable {
 
-    private final String tableName;
-    private final ObjList columns;
     private final CharSequenceIntHashMap columnNameToIndex;
-    private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access
+    private final ObjList columns;
+    private final String tableName;
+    private QwpColumnDef[] cachedColumnDefs;
     private int columnAccessCursor; // tracks expected next column index
+    private boolean columnDefsCacheValid;
+    private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access
     private int rowCount;
     private long schemaHash;
     private boolean schemaHashComputed;
-    private QwpColumnDef[] cachedColumnDefs;
-    private boolean columnDefsCacheValid;
 
     public QwpTableBuffer(String tableName) {
         this.tableName = tableName;
@@ -156,7 +156,7 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable)
             if (candidate.name.equals(name)) {
                 columnAccessCursor++;
                 if (candidate.type != type) {
-                    throw new IllegalArgumentException(
+                    throw new LineSenderException(
                             "Column type mismatch for " + name + ": existing=" + candidate.type + " new=" + type
                     );
                 }
@@ -169,7 +169,7 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable)
         if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) {
             ColumnBuffer existing = columns.get(idx);
             if (existing.type != type) {
-                throw new IllegalArgumentException(
+                throw new LineSenderException(
                         "Column type mismatch for " + name + ": existing=" + existing.type + " new=" + type
                 );
             }
@@ -290,6 +290,66 @@ static int elementSize(byte type) {
         }
     }
 
+    /**
+     * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr().
+     */
+    private static class ArrayCapture implements ArrayBufferAppender {
+        double[] doubleData;
+        int doubleDataOffset;
+        long[] longData;
+        int longDataOffset;
+        byte nDims;
+        int[] shape = new int[32];
+        int shapeIndex;
+
+        @Override
+        public void putBlockOfBytes(long from, long len) {
+            int count = (int) (len / 8);
+            if (doubleData == null) {
+                doubleData = new double[count];
+            }
+            for (int i = 0; i < count; i++) {
+                doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
+            }
+        }
+
+        @Override
+        public void putByte(byte b) {
+            if (shapeIndex == 0) {
+                nDims = b;
+            }
+        }
+
+        @Override
+        public void putDouble(double value) {
+            if (doubleData != null && doubleDataOffset < doubleData.length) {
+                doubleData[doubleDataOffset++] = value;
+            }
+        }
+
+        @Override
+        public void putInt(int value) {
+            if (shapeIndex < nDims) {
+                shape[shapeIndex++] = value;
+                if (shapeIndex == nDims) {
+                    int totalElements = 1;
+                    for (int i = 0; i < nDims; i++) {
+                        totalElements *= shape[i];
+                    }
+                    doubleData = new double[totalElements];
+                    longData = new long[totalElements];
+                }
+            }
+        }
+
+        @Override
+        public void putLong(long value) {
+            if (longData != null && longDataOffset < longData.length) {
+                longData[longDataOffset++] = value;
+            }
+        }
+    }
+
     /**
      * Column buffer for a single column.
      * 

@@ -297,47 +357,37 @@ static int elementSize(byte type) {
      * operation and efficient bulk copy to network buffers.
      */
     public static class ColumnBuffer implements QuietCloseable {
+        final int elemSize;
         final String name;
-        final byte type;
         final boolean nullable;
-        final int elemSize;
-
-        private int size;         // Total row count (including nulls)
-        private int valueCount;   // Actual stored values (excludes nulls)
-
-        // Off-heap data buffer for fixed-width types
-        private OffHeapAppendMemory dataBuffer;
-
-        // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only)
-        private OffHeapAppendMemory auxBuffer;
-
-        // Off-heap null bitmap (bit-packed, 1 bit per row)
-        private long nullBufPtr;
-        private int nullBufCapRows;
-        private boolean hasNulls;
-
-        // On-heap capacity for variable-width arrays (string values, array dims)
-        private int onHeapCapacity;
-
-        // On-heap storage for variable-width types
-        private String[] stringValues;
-
+        final byte type;
+        private final Decimal256 rescaleTemp = new Decimal256();
+        private int arrayDataOffset;
         // Array storage (double/long arrays - variable length per row)
         private byte[] arrayDims;
-        private int[] arrayShapes;
         private int arrayShapeOffset;
+        private int[] arrayShapes;
+        // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only)
+        private OffHeapAppendMemory auxBuffer;
+        // Off-heap data buffer for fixed-width types
+        private OffHeapAppendMemory dataBuffer;
+        // Decimal storage
+        private byte decimalScale = -1;
         private double[] doubleArrayData;
+        private boolean hasNulls;
         private long[] longArrayData;
-        private int arrayDataOffset;
-
+        private int maxGlobalSymbolId = -1;
+        private int nullBufCapRows;
+        // Off-heap null bitmap (bit-packed, 1 bit per row)
+        private long nullBufPtr;
+        private int size;         // Total row count (including nulls)
+        private OffHeapAppendMemory stringData;
+        // Off-heap storage for string/varchar column data
+        private OffHeapAppendMemory stringOffsets;
         // Symbol specific (dictionary stays on-heap)
         private CharSequenceIntHashMap symbolDict;
         private ObjList symbolList;
-        private int maxGlobalSymbolId = -1;
-
-        // Decimal storage
-        private byte decimalScale = -1;
-        private final Decimal256 rescaleTemp = new Decimal256();
+        private int valueCount;   // Actual stored values (excludes nulls)
 
         public ColumnBuffer(String name, byte type, boolean nullable) {
             this.name = name;
@@ -347,7 +397,6 @@ public ColumnBuffer(String name, byte type, boolean nullable) {
             this.size = 0;
             this.valueCount = 0;
             this.hasNulls = false;
-            this.onHeapCapacity = 16;
 
             allocateStorage(type);
             if (nullable) {
@@ -358,12 +407,14 @@ public ColumnBuffer(String name, byte type, boolean nullable) {
         }
 
         public void addBoolean(boolean value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putByte(value ? (byte) 1 : (byte) 0);
             valueCount++;
             size++;
         }
 
         public void addByte(byte value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putByte(value);
             valueCount++;
             size++;
@@ -374,6 +425,7 @@ public void addDecimal128(Decimal128 value) {
                 addNull();
                 return;
             }
+            ensureNullBitmapForNonNull();
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
             } else if (decimalScale != value.getScale()) {
@@ -397,6 +449,7 @@ public void addDecimal256(Decimal256 value) {
                 addNull();
                 return;
             }
+            ensureNullBitmapForNonNull();
             Decimal256 src = value;
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
@@ -418,6 +471,7 @@ public void addDecimal64(Decimal64 value) {
                 addNull();
                 return;
             }
+            ensureNullBitmapForNonNull();
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
                 dataBuffer.putLong(value.getValue());
@@ -434,6 +488,7 @@ public void addDecimal64(Decimal64 value) {
         }
 
         public void addDouble(double value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putDouble(value);
             valueCount++;
             size++;
@@ -534,24 +589,28 @@ public void addDoubleArray(DoubleArray array) {
         }
 
         public void addFloat(float value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putFloat(value);
             valueCount++;
             size++;
         }
 
         public void addInt(int value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putInt(value);
             valueCount++;
             size++;
         }
 
         public void addLong(long value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putLong(value);
             valueCount++;
             size++;
         }
 
         public void addLong256(long l0, long l1, long l2, long l3) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putLong(l0);
             dataBuffer.putLong(l1);
             dataBuffer.putLong(l2);
@@ -689,8 +748,7 @@ public void addNull() {
                         break;
                     case TYPE_STRING:
                     case TYPE_VARCHAR:
-                        ensureOnHeapCapacity();
-                        stringValues[valueCount] = null;
+                        stringOffsets.putInt((int) stringData.getAppendOffset());
                         break;
                     case TYPE_SYMBOL:
                         dataBuffer.putInt(-1);
@@ -725,6 +783,7 @@ public void addNull() {
         }
 
         public void addShort(short value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putShort(value);
             valueCount++;
             size++;
@@ -734,12 +793,15 @@ public void addString(String value) {
             if (value == null && nullable) {
                 ensureNullCapacity(size + 1);
                 markNull(size);
-                size++;
             } else {
-                ensureOnHeapCapacity();
-                stringValues[valueCount++] = value;
-                size++;
+                ensureNullBitmapForNonNull();
+                if (value != null) {
+                    stringData.putUtf8(value);
+                }
+                stringOffsets.putInt((int) stringData.getAppendOffset());
+                valueCount++;
             }
+            size++;
         }
 
         public void addSymbol(String value) {
@@ -748,8 +810,8 @@ public void addSymbol(String value) {
                     ensureNullCapacity(size + 1);
                     markNull(size);
                 }
-                size++;
             } else {
+                ensureNullBitmapForNonNull();
                 int idx = symbolDict.get(value);
                 if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
                     idx = symbolList.size();
@@ -758,8 +820,8 @@ public void addSymbol(String value) {
                 }
                 dataBuffer.putInt(idx);
                 valueCount++;
-                size++;
             }
+            size++;
         }
 
         public void addSymbolWithGlobalId(String value, int globalId) {
@@ -770,6 +832,7 @@ public void addSymbolWithGlobalId(String value, int globalId) {
                 }
                 size++;
             } else {
+                ensureNullBitmapForNonNull();
                 int localIdx = symbolDict.get(value);
                 if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
                     localIdx = symbolList.size();
@@ -793,6 +856,7 @@ public void addSymbolWithGlobalId(String value, int globalId) {
         }
 
         public void addUuid(long high, long low) {
+            ensureNullBitmapForNonNull();
             // Store in wire order: lo first, hi second
             dataBuffer.putLong(low);
             dataBuffer.putLong(high);
@@ -810,6 +874,14 @@ public void close() {
                 auxBuffer.close();
                 auxBuffer = null;
             }
+            if (stringOffsets != null) {
+                stringOffsets.close();
+                stringOffsets = null;
+            }
+            if (stringData != null) {
+                stringData.close();
+                stringData = null;
+            }
             if (nullBufPtr != 0) {
                 Unsafe.free(nullBufPtr, (long) nullBufCapRows >>> 3, MemoryTag.NATIVE_ILP_RSS);
                 nullBufPtr = 0;
@@ -825,14 +897,14 @@ public byte[] getArrayDims() {
             return arrayDims;
         }
 
-        public int[] getArrayShapes() {
-            return arrayShapes;
-        }
-
         public int getArrayShapeOffset() {
             return arrayShapeOffset;
         }
 
+        public int[] getArrayShapes() {
+            return arrayShapes;
+        }
+
         /**
          * Returns the off-heap address of the auxiliary data buffer (global symbol IDs).
          * Returns 0 if no auxiliary data exists.
@@ -883,28 +955,20 @@ public long getNullBitmapAddress() {
             return nullBufPtr;
         }
 
-        /**
-         * Returns the bit-packed null bitmap as a long array.
-         * This creates a new array from off-heap data.
-         */
-        public long[] getNullBitmapPacked() {
-            if (nullBufPtr == 0) {
-                return null;
-            }
-            int longCount = (size + 63) >>> 6;
-            long[] result = new long[longCount];
-            for (int i = 0; i < longCount; i++) {
-                result[i] = Unsafe.getUnsafe().getLong(nullBufPtr + (long) i * 8);
-            }
-            return result;
-        }
-
         public int getSize() {
             return size;
         }
 
-        public String[] getStringValues() {
-            return stringValues;
+        public long getStringDataAddress() {
+            return stringData != null ? stringData.pageAddress() : 0;
+        }
+
+        public long getStringDataSize() {
+            return stringData != null ? stringData.getAppendOffset() : 0;
+        }
+
+        public long getStringOffsetsAddress() {
+            return stringOffsets != null ? stringOffsets.pageAddress() : 0;
         }
 
         public String[] getSymbolDictionary() {
@@ -953,6 +1017,13 @@ public void reset() {
             if (auxBuffer != null) {
                 auxBuffer.truncate();
             }
+            if (stringOffsets != null) {
+                stringOffsets.truncate();
+                stringOffsets.putInt(0); // re-seed initial 0 offset
+            }
+            if (stringData != null) {
+                stringData.truncate();
+            }
             if (nullBufPtr != 0) {
                 Vect.memset(nullBufPtr, (long) nullBufCapRows >>> 3, 0);
             }
@@ -1003,6 +1074,13 @@ public void truncateTo(int newSize) {
                 dataBuffer.jumpTo((long) newValueCount * elemSize);
             }
 
+            // Rewind string buffers
+            if (stringOffsets != null) {
+                int dataOffset = Unsafe.getUnsafe().getInt(stringOffsets.pageAddress() + (long) newValueCount * 4);
+                stringData.jumpTo(dataOffset);
+                stringOffsets.jumpTo((long) (newValueCount + 1) * 4);
+            }
+
             // Rewind aux buffer (symbol global IDs)
             if (auxBuffer != null) {
                 auxBuffer.jumpTo((long) newValueCount * 4);
@@ -1036,7 +1114,9 @@ private void allocateStorage(byte type) {
                     break;
                 case TYPE_STRING:
                 case TYPE_VARCHAR:
-                    stringValues = new String[onHeapCapacity];
+                    stringOffsets = new OffHeapAppendMemory(64);
+                    stringOffsets.putInt(0); // seed initial 0 offset
+                    stringData = new OffHeapAppendMemory(256);
                     break;
                 case TYPE_SYMBOL:
                     dataBuffer = new OffHeapAppendMemory(64);
@@ -1051,7 +1131,7 @@ private void allocateStorage(byte type) {
                     break;
                 case TYPE_DOUBLE_ARRAY:
                 case TYPE_LONG_ARRAY:
-                    arrayDims = new byte[onHeapCapacity];
+                    arrayDims = new byte[16];
                     break;
                 case TYPE_DECIMAL64:
                     dataBuffer = new OffHeapAppendMemory(128);
@@ -1101,6 +1181,12 @@ private void ensureArrayCapacity(int nDims, int dataElements) {
             }
         }
 
+        private void ensureNullBitmapForNonNull() {
+            if (nullBufPtr != 0) {
+                ensureNullCapacity(size + 1);
+            }
+        }
+
         private void ensureNullCapacity(int rows) {
             if (rows > nullBufCapRows) {
                 int newCapRows = Math.max(nullBufCapRows * 2, ((rows + 63) >>> 6) << 6);
@@ -1112,16 +1198,6 @@ private void ensureNullCapacity(int rows) {
             }
         }
 
-        private void ensureOnHeapCapacity() {
-            if (valueCount >= onHeapCapacity) {
-                int newCapacity = onHeapCapacity * 2;
-                if (stringValues != null) {
-                    stringValues = Arrays.copyOf(stringValues, newCapacity);
-                }
-                onHeapCapacity = newCapacity;
-            }
-        }
-
         private void markNull(int index) {
             long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8;
             int bitIndex = index & 63;
@@ -1130,64 +1206,4 @@ private void markNull(int index) {
             hasNulls = true;
         }
     }
-
-    /**
-     * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr().
-     */
-    private static class ArrayCapture implements ArrayBufferAppender {
-        byte nDims;
-        int[] shape = new int[32];
-        int shapeIndex;
-        double[] doubleData;
-        int doubleDataOffset;
-        long[] longData;
-        int longDataOffset;
-
-        @Override
-        public void putBlockOfBytes(long from, long len) {
-            int count = (int) (len / 8);
-            if (doubleData == null) {
-                doubleData = new double[count];
-            }
-            for (int i = 0; i < count; i++) {
-                doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
-            }
-        }
-
-        @Override
-        public void putByte(byte b) {
-            if (shapeIndex == 0) {
-                nDims = b;
-            }
-        }
-
-        @Override
-        public void putDouble(double value) {
-            if (doubleData != null && doubleDataOffset < doubleData.length) {
-                doubleData[doubleDataOffset++] = value;
-            }
-        }
-
-        @Override
-        public void putInt(int value) {
-            if (shapeIndex < nDims) {
-                shape[shapeIndex++] = value;
-                if (shapeIndex == nDims) {
-                    int totalElements = 1;
-                    for (int i = 0; i < nDims; i++) {
-                        totalElements *= shape[i];
-                    }
-                    doubleData = new double[totalElements];
-                    longData = new long[totalElements];
-                }
-            }
-        }
-
-        @Override
-        public void putLong(long value) {
-            if (longData != null && longDataOffset < longData.length) {
-                longData[longDataOffset++] = value;
-            }
-        }
-    }
 }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java
index 96c39a3..073bf27 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java
@@ -263,4 +263,81 @@ public void testLargeGrowth() {
         long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS);
         assertEquals(before, after);
     }
+
+    @Test
+    public void testPutUtf8Ascii() {
+        long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS);
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            mem.putUtf8("hello");
+            assertEquals(5, mem.getAppendOffset());
+            assertEquals('h', Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals('e', Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+            assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(2)));
+            assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(3)));
+            assertEquals('o', Unsafe.getUnsafe().getByte(mem.addressOf(4)));
+        }
+        long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS);
+        assertEquals(before, after);
+    }
+
+    @Test
+    public void testPutUtf8Empty() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            mem.putUtf8("");
+            assertEquals(0, mem.getAppendOffset());
+        }
+    }
+
+    @Test
+    public void testPutUtf8MultiByte() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // 2-byte: U+00E9 (e-acute) = C3 A9
+            mem.putUtf8("\u00E9");
+            assertEquals(2, mem.getAppendOffset());
+            assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals((byte) 0xA9, Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+        }
+    }
+
+    @Test
+    public void testPutUtf8Null() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            mem.putUtf8(null);
+            assertEquals(0, mem.getAppendOffset());
+        }
+    }
+
+    @Test
+    public void testPutUtf8SurrogatePairs() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // U+1F600 (grinning face) = F0 9F 98 80
+            mem.putUtf8("\uD83D\uDE00");
+            assertEquals(4, mem.getAppendOffset());
+            assertEquals((byte) 0xF0, Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals((byte) 0x9F, Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+            assertEquals((byte) 0x98, Unsafe.getUnsafe().getByte(mem.addressOf(2)));
+            assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(mem.addressOf(3)));
+        }
+    }
+
+    @Test
+    public void testPutUtf8ThreeByte() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // 3-byte: U+4E16 (CJK character) = E4 B8 96
+            mem.putUtf8("\u4E16");
+            assertEquals(3, mem.getAppendOffset());
+            assertEquals((byte) 0xE4, Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals((byte) 0xB8, Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+            assertEquals((byte) 0x96, Unsafe.getUnsafe().getByte(mem.addressOf(2)));
+        }
+    }
+
+    @Test
+    public void testPutUtf8Mixed() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes
+            mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00");
+            assertEquals(10, mem.getAppendOffset());
+        }
+    }
 }

From 332b66f44436c3d2750f37acdd3ea21a14e85ad4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 11:22:03 +0100
Subject: [PATCH 017/230] Fix pong frame clobbering in-progress send buffer

sendPongFrame() used the shared sendBuffer, calling reset()
which destroyed any partially-built frame the caller had in
progress via getSendBuffer(). This could happen when a PING
arrived during receiveFrame()/tryReceiveFrame() while the
caller was mid-way through constructing a data frame.

Add a dedicated 256-byte controlFrameBuffer for sending pong
responses. RFC 6455 limits control frame payloads to 125 bytes
plus a 14-byte max header, so 256 bytes is sufficient and never
needs to grow.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index ab8f696..cfe53dc 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -76,6 +76,7 @@ public abstract class WebSocketClient implements QuietCloseable {
     protected final Socket socket;
 
     private final WebSocketSendBuffer sendBuffer;
+    private final WebSocketSendBuffer controlFrameBuffer;
     private final WebSocketFrameParser frameParser;
     private final Rnd rnd;
     private final int defaultTimeout;
@@ -103,6 +104,10 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock
         int sendBufSize = Math.max(configuration.getInitialRequestBufferSize(), DEFAULT_SEND_BUFFER_SIZE);
         int maxSendBufSize = Math.max(configuration.getMaximumRequestBufferSize(), sendBufSize);
         this.sendBuffer = new WebSocketSendBuffer(sendBufSize, maxSendBufSize);
+        // Control frames (ping/pong/close) have max 125-byte payload + 14-byte header.
+        // This dedicated buffer prevents sendPongFrame from clobbering an in-progress
+        // frame being built in the main sendBuffer.
+        this.controlFrameBuffer = new WebSocketSendBuffer(256, 256);
 
         this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE);
         this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT);
@@ -131,6 +136,7 @@ public void close() {
 
             disconnect();
             sendBuffer.close();
+            controlFrameBuffer.close();
 
             if (recvBufPtr != 0) {
                 Unsafe.free(recvBufPtr, recvBufSize, MemoryTag.NATIVE_DEFAULT);
@@ -641,10 +647,10 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
 
     private void sendPongFrame(long payloadPtr, int payloadLen) {
         try {
-            sendBuffer.reset();
-            WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePongFrame(payloadPtr, payloadLen);
-            doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, 1000); // Short timeout for pong
-            sendBuffer.reset();
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
         } catch (Exception e) {
             LOG.error("Failed to send pong: {}", e.getMessage());
         }

From 82955d057840c692181fd03402ff5e386cd7665e Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:04:38 +0100
Subject: [PATCH 018/230] Fix buffer overrun in WebSocket close frame

sendCloseFrame() used reason.length() (UTF-16 code units) to
calculate the payload size, but wrote reason.getBytes(UTF_8)
(UTF-8 bytes) into the buffer. For non-ASCII close reasons,
UTF-8 encoding can be longer than the UTF-16 length, causing
writes past the declared payload size. This corrupted the
frame header length, the masking range, and could overrun the
allocated buffer.

Compute the UTF-8 byte array upfront and use its length for
all sizing calculations.

Co-Authored-By: Claude Opus 4.6 
---
 .../client/cutlass/qwp/client/WebSocketChannel.java        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
index 8774ac2..f9584f4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
@@ -429,7 +429,9 @@ private void sendCloseFrame(int code, String reason) {
         int maskKey = rnd.nextInt();
 
         // Close payload: 2-byte code + optional reason
-        int reasonLen = (reason != null) ? reason.length() : 0;
+        // Compute UTF-8 bytes upfront so payload length is correct
+        byte[] reasonBytes = (reason != null) ? reason.getBytes(StandardCharsets.UTF_8) : null;
+        int reasonLen = (reasonBytes != null) ? reasonBytes.length : 0;
         int payloadLen = 2 + reasonLen;
 
         int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
@@ -447,8 +449,7 @@ private void sendCloseFrame(int code, String reason) {
         Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF));
 
         // Write reason if present
-        if (reason != null) {
-            byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8);
+        if (reasonBytes != null) {
             for (int i = 0; i < reasonBytes.length; i++) {
                 Unsafe.getUnsafe().putByte(payloadStart + 2 + i, reasonBytes[i]);
             }

From 8439f45581e9f9c4454c5c01c932994c4dc84af7 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:10:55 +0100
Subject: [PATCH 019/230] Echo close frame on WebSocket close (RFC 6455)

When receiving a CLOSE frame from the server, the client now echoes
a close frame back before marking the connection as no longer
upgraded. This is required by RFC 6455 Section 5.5.1.

The close code parsing was moved out of the handler-null check so
the code is always available for the echo. The echo uses the
dedicated controlFrameBuffer to avoid clobbering any in-progress
frame in the main send buffer.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java  | 40 +++++++++++++------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index cfe53dc..ddc4a02 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -603,21 +603,24 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
                     }
                     break;
                 case WebSocketOpcode.CLOSE:
-                    upgraded = false;
-                    if (handler != null) {
-                        int closeCode = 0;
-                        String reason = null;
-                        if (payloadLen >= 2) {
-                            closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8)
-                                    | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF);
-                            if (payloadLen > 2) {
-                                byte[] reasonBytes = new byte[payloadLen - 2];
-                                for (int i = 0; i < reasonBytes.length; i++) {
-                                    reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i);
-                                }
-                                reason = new String(reasonBytes, StandardCharsets.UTF_8);
+                    int closeCode = 0;
+                    String reason = null;
+                    if (payloadLen >= 2) {
+                        closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8)
+                                | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF);
+                        if (payloadLen > 2) {
+                            byte[] reasonBytes = new byte[payloadLen - 2];
+                            for (int i = 0; i < reasonBytes.length; i++) {
+                                reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i);
                             }
+                            reason = new String(reasonBytes, StandardCharsets.UTF_8);
                         }
+                    }
+                    // RFC 6455 Section 5.5.1: echo a close frame back before
+                    // marking the connection as no longer upgraded
+                    sendCloseFrameEcho(closeCode);
+                    upgraded = false;
+                    if (handler != null) {
                         handler.onClose(closeCode, reason);
                     }
                     break;
@@ -645,6 +648,17 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
         return false;
     }
 
+    private void sendCloseFrameEcho(int code) {
+        try {
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
+        } catch (Exception e) {
+            LOG.error("Failed to echo close frame: {}", e.getMessage());
+        }
+    }
+
     private void sendPongFrame(long payloadPtr, int payloadLen) {
         try {
             controlFrameBuffer.reset();

From c41aa58b8e112d2ddc60f3cdc2c0ff996f7403c1 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:15:38 +0100
Subject: [PATCH 020/230] Add WebSocket fragmentation support (RFC 6455)

Handle CONTINUATION frames (opcode 0x0) in tryParseFrame()
which were previously silently dropped. Fragment payloads are
accumulated in a lazily-allocated native memory buffer and
delivered as a complete message to the handler when the final
FIN=1 frame arrives.

The FIN bit is now checked on TEXT/BINARY frames: FIN=0 starts
fragment accumulation, FIN=1 delivers immediately. Protocol
errors are raised for continuation without an initial fragment
and for overlapping fragmented messages.

The fragment buffer is freed in close() and the fragmentation
state is reset on disconnect().

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java  | 71 +++++++++++++++++--
 1 file changed, 66 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index ddc4a02..804c540 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -93,6 +93,12 @@ public abstract class WebSocketClient implements QuietCloseable {
     private boolean upgraded;
     private boolean closed;
 
+    // Fragmentation state (RFC 6455 Section 5.4)
+    private int fragmentOpcode = -1;   // opcode of first fragment, -1 = not in a fragmented message
+    private long fragmentBufPtr;       // native buffer for accumulating fragment payloads
+    private int fragmentBufSize;
+    private int fragmentBufPos;
+
     // Handshake key for verification
     private String handshakeKey;
 
@@ -138,6 +144,11 @@ public void close() {
             sendBuffer.close();
             controlFrameBuffer.close();
 
+            if (fragmentBufPtr != 0) {
+                Unsafe.free(fragmentBufPtr, fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
+                fragmentBufPtr = 0;
+            }
+
             if (recvBufPtr != 0) {
                 Unsafe.free(recvBufPtr, recvBufSize, MemoryTag.NATIVE_DEFAULT);
                 recvBufPtr = 0;
@@ -156,6 +167,7 @@ public void disconnect() {
         port = 0;
         recvPos = 0;
         recvReadPos = 0;
+        resetFragmentState();
     }
 
     /**
@@ -625,13 +637,40 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
                     }
                     break;
                 case WebSocketOpcode.BINARY:
-                    if (handler != null) {
-                        handler.onBinaryMessage(payloadPtr, payloadLen);
+                case WebSocketOpcode.TEXT:
+                    if (frameParser.isFin()) {
+                        if (fragmentOpcode != -1) {
+                            throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message");
+                        }
+                        if (handler != null) {
+                            if (opcode == WebSocketOpcode.BINARY) {
+                                handler.onBinaryMessage(payloadPtr, payloadLen);
+                            } else {
+                                handler.onTextMessage(payloadPtr, payloadLen);
+                            }
+                        }
+                    } else {
+                        if (fragmentOpcode != -1) {
+                            throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message");
+                        }
+                        fragmentOpcode = opcode;
+                        appendToFragmentBuffer(payloadPtr, payloadLen);
                     }
                     break;
-                case WebSocketOpcode.TEXT:
-                    if (handler != null) {
-                        handler.onTextMessage(payloadPtr, payloadLen);
+                case WebSocketOpcode.CONTINUATION:
+                    if (fragmentOpcode == -1) {
+                        throw new HttpClientException("WebSocket protocol error: continuation frame without initial fragment");
+                    }
+                    appendToFragmentBuffer(payloadPtr, payloadLen);
+                    if (frameParser.isFin()) {
+                        if (handler != null) {
+                            if (fragmentOpcode == WebSocketOpcode.BINARY) {
+                                handler.onBinaryMessage(fragmentBufPtr, fragmentBufPos);
+                            } else {
+                                handler.onTextMessage(fragmentBufPtr, fragmentBufPos);
+                            }
+                        }
+                        resetFragmentState();
                     }
                     break;
             }
@@ -670,6 +709,28 @@ private void sendPongFrame(long payloadPtr, int payloadLen) {
         }
     }
 
+    private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
+        if (payloadLen == 0) {
+            return;
+        }
+        int required = fragmentBufPos + payloadLen;
+        if (fragmentBufPtr == 0) {
+            fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
+            fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
+        } else if (required > fragmentBufSize) {
+            int newSize = Math.max(fragmentBufSize * 2, required);
+            fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+            fragmentBufSize = newSize;
+        }
+        Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen);
+        fragmentBufPos += payloadLen;
+    }
+
+    private void resetFragmentState() {
+        fragmentOpcode = -1;
+        fragmentBufPos = 0;
+    }
+
     private void compactRecvBuffer() {
         if (recvReadPos > 0) {
             int remaining = recvPos - recvReadPos;

From 8457d1b333d8678e41aa69eff6323c5892c5518a Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:34:29 +0100
Subject: [PATCH 021/230] Cap recv buffer growth to prevent OOM

Add a configurable maximum size for the WebSocket receive buffer,
mirroring the pattern already used by WebSocketSendBuffer. Previously,
growRecvBuffer() doubled the buffer without any upper bound, allowing
a malicious server to trigger out-of-memory by sending arbitrarily
large frames.

Add getMaximumResponseBufferSize() to HttpClientConfiguration
(defaulting to Integer.MAX_VALUE for backwards compatibility) and
enforce the limit in both growRecvBuffer() and
appendToFragmentBuffer(), which had the same unbounded growth issue
for fragmented messages.

Co-Authored-By: Claude Opus 4.6 
---
 .../client/HttpClientConfiguration.java       |  4 ++++
 .../cutlass/http/client/WebSocketClient.java  | 21 ++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/io/questdb/client/HttpClientConfiguration.java b/core/src/main/java/io/questdb/client/HttpClientConfiguration.java
index c2f644e..b02d485 100644
--- a/core/src/main/java/io/questdb/client/HttpClientConfiguration.java
+++ b/core/src/main/java/io/questdb/client/HttpClientConfiguration.java
@@ -54,6 +54,10 @@ default int getMaximumRequestBufferSize() {
         return Integer.MAX_VALUE;
     }
 
+    default int getMaximumResponseBufferSize() {
+        return Integer.MAX_VALUE;
+    }
+
     default NetworkFacade getNetworkFacade() {
         return NetworkFacadeImpl.INSTANCE;
     }
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index 804c540..5028dcf 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -80,6 +80,7 @@ public abstract class WebSocketClient implements QuietCloseable {
     private final WebSocketFrameParser frameParser;
     private final Rnd rnd;
     private final int defaultTimeout;
+    private final int maxRecvBufSize;
 
     // Receive buffer (native memory)
     private long recvBufPtr;
@@ -116,6 +117,7 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock
         this.controlFrameBuffer = new WebSocketSendBuffer(256, 256);
 
         this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE);
+        this.maxRecvBufSize = Math.max(configuration.getMaximumResponseBufferSize(), recvBufSize);
         this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT);
         this.recvPos = 0;
         this.recvReadPos = 0;
@@ -714,11 +716,18 @@ private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
             return;
         }
         int required = fragmentBufPos + payloadLen;
+        if (required > maxRecvBufSize) {
+            throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=")
+                    .put(required)
+                    .put(", max=")
+                    .put(maxRecvBufSize)
+                    .put(']');
+        }
         if (fragmentBufPtr == 0) {
             fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
             fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
         } else if (required > fragmentBufSize) {
-            int newSize = Math.max(fragmentBufSize * 2, required);
+            int newSize = Math.min(Math.max(fragmentBufSize * 2, required), maxRecvBufSize);
             fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
             fragmentBufSize = newSize;
         }
@@ -744,6 +753,16 @@ private void compactRecvBuffer() {
 
     private void growRecvBuffer() {
         int newSize = recvBufSize * 2;
+        if (newSize > maxRecvBufSize) {
+            if (recvBufSize >= maxRecvBufSize) {
+                throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=")
+                        .put(recvBufSize)
+                        .put(", max=")
+                        .put(maxRecvBufSize)
+                        .put(']');
+            }
+            newSize = maxRecvBufSize;
+        }
         recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
         recvBufSize = newSize;
     }

From 1dc87d71d7b39a0b9cdda4da3a527099dd87a7bc Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:48:31 +0100
Subject: [PATCH 022/230] Use ephemeral ports in WebSocket builder tests

Tests that expect connection failure were hardcoding ports
(9000, 19999) which could collide with running services. When
a QuestDB server is running on port 9000, the WebSocket
connection succeeds and the test fails with "Expected
LineSenderException".

Replace hardcoded ports with dynamically allocated ephemeral
ports via ServerSocket(0). The port is bound and immediately
closed, guaranteeing nothing is listening when the test tries
to connect.

Affected tests:
- testBuilderWithWebSocketTransportCreatesCorrectSenderType
- testConnectionRefused
- testWsConfigString
- testWsConfigString_missingAddr_fails
- testWsConfigString_protocolAlreadyConfigured_fails

Co-Authored-By: Claude Opus 4.6 
---
 .../LineSenderBuilderWebSocketTest.java       | 34 +++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
index ee80665..4fb8121 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
@@ -252,19 +252,24 @@ public void testBuilderWithWebSocketTransport() {
     }
 
     @Test
-    public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() {
+    public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() throws Exception {
+        int port;
+        try (java.net.ServerSocket s = new java.net.ServerSocket(0)) {
+            port = s.getLocalPort();
+        }
         assertThrowsAny(
                 Sender.builder(Sender.Transport.WEBSOCKET)
-                        .address(LOCALHOST + ":9000"),
+                        .address(LOCALHOST + ":" + port),
                 "connect", "Failed"
         );
     }
 
     @Test
-    public void testConnectionRefused() {
+    public void testConnectionRefused() throws Exception {
+        int port = findUnusedPort();
         assertThrowsAny(
                 Sender.builder(Sender.Transport.WEBSOCKET)
-                        .address(LOCALHOST + ":19999"),
+                        .address(LOCALHOST + ":" + port),
                 "connect", "Failed"
         );
     }
@@ -691,22 +696,25 @@ public void testUsernamePassword_notYetSupported() {
     }
 
     @Test
-    public void testWsConfigString() {
-        assertBadConfig("ws::addr=localhost:9000;", "connect", "Failed");
+    public void testWsConfigString() throws Exception {
+        int port = findUnusedPort();
+        assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed");
     }
 
     // ==================== Edge Cases ====================
 
     @Test
-    public void testWsConfigString_missingAddr_fails() {
-        assertBadConfig("ws::addr=localhost;", "connect", "Failed");
+    public void testWsConfigString_missingAddr_fails() throws Exception {
+        int port = findUnusedPort();
+        assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed");
         assertBadConfig("ws::foo=bar;", "addr is missing");
     }
 
     @Test
-    public void testWsConfigString_protocolAlreadyConfigured_fails() {
+    public void testWsConfigString_protocolAlreadyConfigured_fails() throws Exception {
+        int port = findUnusedPort();
         assertThrowsAny(
-                Sender.builder("ws::addr=localhost:9000;")
+                Sender.builder("ws::addr=localhost:" + port + ";")
                         .enableTls(),
                 "TLS", "connect", "Failed"
         );
@@ -761,6 +769,12 @@ private static void assertThrowsAny(Sender.LineSenderBuilder builder, String...
         assertThrowsAny(builder::build, anyOf);
     }
 
+    private static int findUnusedPort() throws Exception {
+        try (java.net.ServerSocket s = new java.net.ServerSocket(0)) {
+            return s.getLocalPort();
+        }
+    }
+
     private static void assertThrowsAny(Runnable action, String... anyOf) {
         try {
             action.run();

From 00c145b98372c4eb16edd7ebb55f67b7966fe1ff Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 13:35:40 +0100
Subject: [PATCH 023/230] Auto-cleanup test code

---
 .../questdb/client/test/AbstractQdbTest.java  | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
index 81242bd..244488f 100644
--- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
+++ b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
@@ -31,7 +31,6 @@
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
@@ -210,10 +209,10 @@ public static void setUpStatic() {
             System.err.printf("CLEANING UP TEST TABLES%n");
             // Cleanup all test tables before starting tests
             try (Connection conn = getPgConnection();
-                    Statement readStmt = conn.createStatement();
-                    Statement stmt = conn.createStatement();
-                    ResultSet rs = readStmt
-                            .executeQuery("SELECT table_name FROM tables() WHERE table_name LIKE 'test_%'")) {
+                 Statement readStmt = conn.createStatement();
+                 Statement stmt = conn.createStatement();
+                 ResultSet rs = readStmt
+                         .executeQuery("SELECT table_name FROM tables() WHERE table_name LIKE 'test_%'")) {
                 while (rs.next()) {
                     String tableName = rs.getString(1);
                     try {
@@ -458,7 +457,7 @@ protected static Connection initPgConnection() throws SQLException {
     protected void assertSqlEventually(CharSequence expected, String sql) throws Exception {
         assertEventually(() -> {
             try (Statement statement = getPgConnection().createStatement();
-                    ResultSet rs = statement.executeQuery(sql)) {
+                 ResultSet rs = statement.executeQuery(sql)) {
                 sink.clear();
                 printToSink(sink, rs);
                 TestUtils.assertEquals(expected, sink);
@@ -474,8 +473,8 @@ protected void assertSqlEventually(CharSequence expected, String sql) throws Exc
     protected void assertTableExistsEventually(CharSequence tableName) throws Exception {
         assertEventually(() -> {
             try (Statement stmt = getPgConnection().createStatement();
-                    ResultSet rs = stmt.executeQuery(
-                            String.format("SELECT COUNT(*) AS cnt FROM tables() WHERE table_name = '%s'", tableName))) {
+                 ResultSet rs = stmt.executeQuery(
+                         String.format("SELECT COUNT(*) AS cnt FROM tables() WHERE table_name = '%s'", tableName))) {
                 Assert.assertTrue(rs.next());
                 final long actualSize = rs.getLong(1);
                 Assert.assertEquals(1, actualSize);
@@ -546,7 +545,7 @@ protected List> executeQuery(String sql) throws SQLException
         List> results = new ArrayList<>();
 
         try (Statement stmt = getPgConnection().createStatement();
-                ResultSet rs = stmt.executeQuery(sql)) {
+             ResultSet rs = stmt.executeQuery(sql)) {
 
             ResultSetMetaData metaData = rs.getMetaData();
             int columnCount = metaData.getColumnCount();
@@ -595,7 +594,7 @@ protected String queryTableAsTsv(String tableName, String orderBy) throws SQLExc
         }
 
         try (Statement stmt = getPgConnection().createStatement();
-                ResultSet rs = stmt.executeQuery(sql)) {
+             ResultSet rs = stmt.executeQuery(sql)) {
 
             ResultSetMetaData metaData = rs.getMetaData();
             int columnCount = metaData.getColumnCount();

From dbca59054609682d5f7987be78cfe9a3e3311f09 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 13:35:56 +0100
Subject: [PATCH 024/230] Don't default QUESTDB_RUNNING to true

---
 core/src/test/java/io/questdb/client/test/AbstractQdbTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
index 244488f..e984f19 100644
--- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
+++ b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
@@ -425,7 +425,7 @@ protected static String getPgUser() {
      * Get whether a QuestDB instance is running locally.
      */
     protected static boolean getQuestDBRunning() {
-        return getConfigBool("QUESTDB_RUNNING", "questdb.running", true);
+        return getConfigBool("QUESTDB_RUNNING", "questdb.running", false);
     }
 
     /**

From 34cc15496cf52bf41bcc958b1bcb7e838f83559e Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 13:41:32 +0100
Subject: [PATCH 025/230] Fix case-sensitive header check in WebSocket
 handshake

The Sec-WebSocket-Accept header validation used case-sensitive
String.contains(), which violates RFC 7230 (HTTP headers are
case-insensitive). A server sending the header in a different
casing (e.g., sec-websocket-accept) would cause the handshake
to fail.

Replace with a containsHeaderValue() helper that uses
String.regionMatches(ignoreCase=true) for the header name
lookup, avoiding both the case-sensitivity bug and unnecessary
string allocation from toLowerCase().

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java  | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index 5028dcf..fec618f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -378,13 +378,30 @@ private void validateUpgradeResponse(int headerEnd) {
             throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine);
         }
 
-        // Verify Sec-WebSocket-Accept
+        // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230)
         String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey);
-        if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) {
+        if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) {
             throw new HttpClientException("Invalid Sec-WebSocket-Accept header");
         }
     }
 
+    private static boolean containsHeaderValue(String response, String headerName, String expectedValue) {
+        int headerLen = headerName.length();
+        int responseLen = response.length();
+        for (int i = 0; i <= responseLen - headerLen; i++) {
+            if (response.regionMatches(true, i, headerName, 0, headerLen)) {
+                int valueStart = i + headerLen;
+                int lineEnd = response.indexOf('\r', valueStart);
+                if (lineEnd < 0) {
+                    lineEnd = responseLen;
+                }
+                String actualValue = response.substring(valueStart, lineEnd).trim();
+                return actualValue.equals(expectedValue);
+            }
+        }
+        return false;
+    }
+
     // === Sending ===
 
     /**

From 2834b03d083a73cf152c3681e1a19381e00adbb7 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:16:20 +0100
Subject: [PATCH 026/230] Use bulk copyMemory for WebSocket I/O

Replace byte-by-byte native-heap copies in writeToSocket and
readFromSocket with Unsafe.copyMemory(), using the 5-argument
form that bridges native memory and Java byte arrays via
Unsafe.BYTE_OFFSET.

Add WebSocketChannelTest with a local echo server that verifies
data integrity through the copy paths across various payload
sizes and patterns.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/client/WebSocketChannel.java  |   8 +-
 .../qwp/client/WebSocketChannelTest.java      | 430 ++++++++++++++++++
 2 files changed, 432 insertions(+), 6 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
index f9584f4..415ee4b 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
@@ -595,9 +595,7 @@ private void writeToSocket(long ptr, int len) throws IOException {
         // Copy to temp array for socket write (unavoidable with OutputStream)
         // Use separate write buffer to avoid race with read thread
         byte[] temp = getWriteTempBuffer(len);
-        for (int i = 0; i < len; i++) {
-            temp[i] = Unsafe.getUnsafe().getByte(ptr + i);
-        }
+        Unsafe.getUnsafe().copyMemory(null, ptr, temp, Unsafe.BYTE_OFFSET, len);
         out.write(temp, 0, len);
         out.flush();
     }
@@ -618,9 +616,7 @@ private int readFromSocket() throws IOException {
         byte[] temp = getReadTempBuffer(available);
         int bytesRead = in.read(temp, 0, available);
         if (bytesRead > 0) {
-            for (int i = 0; i < bytesRead; i++) {
-                Unsafe.getUnsafe().putByte(recvBufferPtr + recvBufferPos + i, temp[i]);
-            }
+            Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead);
             recvBufferPos += bytesRead;
         }
         return bytesRead;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java
new file mode 100644
index 0000000..607b442
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java
@@ -0,0 +1,430 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.WebSocketChannel;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.AbstractTest;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Tests for WebSocketChannel's native-heap memory copy paths.
+ * Exercises writeToSocket (native to heap) and readFromSocket (heap to native)
+ * through a local echo server.
+ */
+public class WebSocketChannelTest extends AbstractTest {
+
+    @Test
+    public void testBinaryRoundTripSmallPayload() throws Exception {
+        TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13));
+    }
+
+    @Test
+    public void testBinaryRoundTripMediumPayload() throws Exception {
+        TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096));
+    }
+
+    @Test
+    public void testBinaryRoundTripLargePayload() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            // Large payload that exercises bulk copyMemory across many cache lines.
+            // Kept under 32KB so the echo response arrives in a single TCP read
+            // on loopback (avoids a pre-existing bug in doReceiveFrame with
+            // partial frame assembly).
+            assertBinaryRoundTrip(30_000);
+        });
+    }
+
+    @Test
+    public void testBinaryRoundTripAllByteValues() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            int len = 256;
+            long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+            try {
+                for (int i = 0; i < len; i++) {
+                    Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i);
+                }
+                assertBinaryRoundTrip(sendPtr, len);
+            } finally {
+                Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
+    @Test
+    public void testBinaryRoundTripRepeatedFrames() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            int payloadLen = 1000;
+            int frameCount = 10;
+            long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+            try (EchoServer server = new EchoServer()) {
+                server.start();
+                WebSocketChannel channel = new WebSocketChannel(
+                        "localhost:" + server.getPort() + "/", false
+                );
+                try {
+                    channel.setConnectTimeout(5000);
+                    channel.setReadTimeout(5000);
+                    channel.connect();
+
+                    for (int f = 0; f < frameCount; f++) {
+                        for (int i = 0; i < payloadLen; i++) {
+                            Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i + f));
+                        }
+                        channel.sendBinary(sendPtr, payloadLen);
+
+                        ReceivedPayload received = new ReceivedPayload();
+                        boolean ok = receiveWithRetry(channel, received, 5000);
+                        server.assertNoError();
+                        Assert.assertTrue("frame " + f + ": expected response", ok);
+                        Assert.assertEquals("frame " + f + ": length", payloadLen, received.length);
+
+                        for (int i = 0; i < payloadLen; i++) {
+                            Assert.assertEquals(
+                                    "frame " + f + " byte " + i,
+                                    (byte) (i + f),
+                                    Unsafe.getUnsafe().getByte(received.ptr + i)
+                            );
+                        }
+                    }
+                } finally {
+                    channel.close();
+                }
+                server.assertNoError();
+            } finally {
+                Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
+    private void assertBinaryRoundTrip(int payloadLen) throws Exception {
+        long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+        try {
+            for (int i = 0; i < payloadLen; i++) {
+                Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i & 0xFF));
+            }
+            assertBinaryRoundTrip(sendPtr, payloadLen);
+        } finally {
+            Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
+    private void assertBinaryRoundTrip(long sendPtr, int payloadLen) throws Exception {
+        try (EchoServer server = new EchoServer()) {
+            server.start();
+            WebSocketChannel channel = new WebSocketChannel(
+                    "localhost:" + server.getPort() + "/", false
+            );
+            try {
+                channel.setConnectTimeout(5000);
+                channel.setReadTimeout(5000);
+                channel.connect();
+
+                // Send exercises writeToSocket (native to heap via copyMemory)
+                channel.sendBinary(sendPtr, payloadLen);
+
+                // Receive exercises readFromSocket (heap to native via copyMemory)
+                ReceivedPayload received = new ReceivedPayload();
+                boolean ok = receiveWithRetry(channel, received, 5000);
+
+                // Check server error before client assertions
+                server.assertNoError();
+                Assert.assertTrue("expected a frame back from echo server", ok);
+                Assert.assertEquals("payload length mismatch", payloadLen, received.length);
+
+                for (int i = 0; i < payloadLen; i++) {
+                    byte expected = Unsafe.getUnsafe().getByte(sendPtr + i);
+                    byte actual = Unsafe.getUnsafe().getByte(received.ptr + i);
+                    Assert.assertEquals("byte mismatch at offset " + i, expected, actual);
+                }
+            } finally {
+                channel.close();
+            }
+            server.assertNoError();
+        }
+    }
+
+    /**
+     * Calls receiveFrame in a loop to handle the case where doReceiveFrame
+     * needs multiple reads to assemble a complete frame (e.g. header and
+     * payload arrive in separate TCP segments).
+     */
+    private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) {
+        long deadline = System.currentTimeMillis() + timeoutMs;
+        while (System.currentTimeMillis() < deadline) {
+            int remaining = (int) (deadline - System.currentTimeMillis());
+            if (remaining <= 0) {
+                break;
+            }
+            if (channel.receiveFrame(handler, remaining)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static class ReceivedPayload implements WebSocketChannel.ResponseHandler {
+        long ptr;
+        int length;
+
+        @Override
+        public void onBinaryMessage(long payload, int length) {
+            this.ptr = payload;
+            this.length = length;
+        }
+
+        @Override
+        public void onClose(int code, String reason) {
+        }
+    }
+
+    /**
+     * Minimal WebSocket echo server. Accepts one connection, completes the
+     * HTTP upgrade handshake, then echoes every binary frame back unmasked.
+     * All echo writes use a single byte array to avoid TCP fragmentation.
+     */
+    private static class EchoServer implements AutoCloseable {
+        private static final Pattern KEY_PATTERN =
+                Pattern.compile("Sec-WebSocket-Key:\\s*(.+?)\\r\\n");
+
+        private final ServerSocket serverSocket;
+        private final AtomicReference error = new AtomicReference<>();
+        private Thread thread;
+
+        EchoServer() throws IOException {
+            serverSocket = new ServerSocket(0);
+        }
+
+        int getPort() {
+            return serverSocket.getLocalPort();
+        }
+
+        void start() {
+            thread = new Thread(this::run, "ws-echo-server");
+            thread.setDaemon(true);
+            thread.start();
+        }
+
+        void assertNoError() {
+            Throwable t = error.get();
+            if (t != null) {
+                throw new AssertionError("echo server error", t);
+            }
+        }
+
+        @Override
+        public void close() throws Exception {
+            serverSocket.close();
+            if (thread != null) {
+                thread.join(5000);
+            }
+        }
+
+        private void run() {
+            try (Socket client = serverSocket.accept()) {
+                client.setSoTimeout(10_000);
+                client.setTcpNoDelay(true);
+                InputStream in = client.getInputStream();
+                OutputStream out = new BufferedOutputStream(client.getOutputStream());
+
+                completeHandshake(in, out);
+                echoFrames(in, out);
+            } catch (IOException e) {
+                if (!serverSocket.isClosed()) {
+                    error.set(e);
+                }
+            } catch (Throwable t) {
+                error.set(t);
+            }
+        }
+
+        private void completeHandshake(InputStream in, OutputStream out) throws IOException {
+            byte[] buf = new byte[4096];
+            int pos = 0;
+
+            while (pos < buf.length) {
+                int b = in.read();
+                if (b < 0) {
+                    throw new IOException("connection closed during handshake");
+                }
+                buf[pos++] = (byte) b;
+                if (pos >= 4
+                        && buf[pos - 4] == '\r' && buf[pos - 3] == '\n'
+                        && buf[pos - 2] == '\r' && buf[pos - 1] == '\n') {
+                    break;
+                }
+            }
+
+            String request = new String(buf, 0, pos, StandardCharsets.US_ASCII);
+            Matcher m = KEY_PATTERN.matcher(request);
+            if (!m.find()) {
+                throw new IOException("no Sec-WebSocket-Key in request:\n" + request);
+            }
+            String clientKey = m.group(1).trim();
+            String acceptKey = WebSocketHandshake.computeAcceptKey(clientKey);
+
+            String response = "HTTP/1.1 101 Switching Protocols\r\n"
+                    + "Upgrade: websocket\r\n"
+                    + "Connection: Upgrade\r\n"
+                    + "Sec-WebSocket-Accept: " + acceptKey + "\r\n"
+                    + "\r\n";
+            out.write(response.getBytes(StandardCharsets.US_ASCII));
+            out.flush();
+        }
+
+        private void echoFrames(InputStream in, OutputStream out) throws IOException {
+            byte[] readBuf = new byte[256 * 1024];
+
+            while (true) {
+                int pos = 0;
+                while (pos < 2) {
+                    int n = in.read(readBuf, pos, readBuf.length - pos);
+                    if (n < 0) {
+                        return;
+                    }
+                    pos += n;
+                }
+
+                int byte0 = readBuf[0] & 0xFF;
+                int byte1 = readBuf[1] & 0xFF;
+                int opcode = byte0 & 0x0F;
+                boolean masked = (byte1 & 0x80) != 0;
+                int lengthField = byte1 & 0x7F;
+
+                int headerSize = 2;
+                long payloadLength;
+                if (lengthField <= 125) {
+                    payloadLength = lengthField;
+                } else if (lengthField == 126) {
+                    while (pos < 4) {
+                        int n = in.read(readBuf, pos, readBuf.length - pos);
+                        if (n < 0) return;
+                        pos += n;
+                    }
+                    payloadLength = ((readBuf[2] & 0xFF) << 8) | (readBuf[3] & 0xFF);
+                    headerSize = 4;
+                } else {
+                    while (pos < 10) {
+                        int n = in.read(readBuf, pos, readBuf.length - pos);
+                        if (n < 0) return;
+                        pos += n;
+                    }
+                    payloadLength = 0;
+                    for (int i = 0; i < 8; i++) {
+                        payloadLength = (payloadLength << 8) | (readBuf[2 + i] & 0xFF);
+                    }
+                    headerSize = 10;
+                }
+
+                if (masked) {
+                    headerSize += 4;
+                }
+
+                int totalFrameSize = (int) (headerSize + payloadLength);
+
+                if (totalFrameSize > readBuf.length) {
+                    byte[] newBuf = new byte[totalFrameSize];
+                    System.arraycopy(readBuf, 0, newBuf, 0, pos);
+                    readBuf = newBuf;
+                }
+
+                while (pos < totalFrameSize) {
+                    int n = in.read(readBuf, pos, totalFrameSize - pos);
+                    if (n < 0) return;
+                    pos += n;
+                }
+
+                if (opcode == WebSocketOpcode.CLOSE) {
+                    return;
+                }
+
+                if (opcode != WebSocketOpcode.BINARY && opcode != WebSocketOpcode.TEXT) {
+                    continue;
+                }
+
+                // Unmask payload in place
+                if (masked) {
+                    int maskKeyOffset = headerSize - 4;
+                    byte m0 = readBuf[maskKeyOffset];
+                    byte m1 = readBuf[maskKeyOffset + 1];
+                    byte m2 = readBuf[maskKeyOffset + 2];
+                    byte m3 = readBuf[maskKeyOffset + 3];
+                    for (int i = 0; i < (int) payloadLength; i++) {
+                        switch (i & 3) {
+                            case 0: readBuf[headerSize + i] ^= m0; break;
+                            case 1: readBuf[headerSize + i] ^= m1; break;
+                            case 2: readBuf[headerSize + i] ^= m2; break;
+                            case 3: readBuf[headerSize + i] ^= m3; break;
+                        }
+                    }
+                }
+
+                // Build complete unmasked response frame in a single array
+                byte[] responseHeader;
+                if (payloadLength <= 125) {
+                    responseHeader = new byte[]{
+                            (byte) (0x80 | opcode),
+                            (byte) payloadLength
+                    };
+                } else if (payloadLength <= 65535) {
+                    responseHeader = new byte[]{
+                            (byte) (0x80 | opcode),
+                            126,
+                            (byte) ((payloadLength >> 8) & 0xFF),
+                            (byte) (payloadLength & 0xFF)
+                    };
+                } else {
+                    responseHeader = new byte[10];
+                    responseHeader[0] = (byte) (0x80 | opcode);
+                    responseHeader[1] = 127;
+                    for (int i = 0; i < 8; i++) {
+                        responseHeader[2 + i] = (byte) ((payloadLength >> (56 - i * 8)) & 0xFF);
+                    }
+                }
+
+                // Single write: header + payload together via BufferedOutputStream
+                out.write(responseHeader);
+                out.write(readBuf, headerSize, (int) payloadLength);
+                out.flush();
+            }
+        }
+    }
+}

From 6a94139fa601a9398374d9d983b4a3e64fcda890 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:29:15 +0100
Subject: [PATCH 027/230] Fix delta dict corruption on send failure

Move maxSentSymbolId and sentSchemaHashes updates to after the
send/enqueue succeeds in both async and sync flush paths. Previously
these were updated before the send, so if sealAndSwapBuffer() threw
(async) or sendBinary()/waitForAck() threw (sync), the next batch's
delta dictionary would omit symbols the server never received,
silently corrupting subsequent data.

Also move sentSchemaHashes.add() inside the messageSize > 0 guard
in the sync path, where it was incorrectly marking schemas as sent
even when no data was produced.

Co-Authored-By: Claude Opus 4.6 
---
 .../qwp/client/QwpWebSocketSender.java        | 35 +++----
 .../qwp/client/QwpDeltaDictRollbackTest.java  | 93 +++++++++++++++++++
 2 files changed, 112 insertions(+), 16 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 68dc19d..cf697d4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -1141,10 +1141,6 @@ private void flushPendingRows() {
                         useSchemaRef
                 );
 
-                // Track schema key if this was the first time sending this schema
-                if (!useSchemaRef) {
-                    sentSchemaHashes.add(schemaKey);
-                }
                 QwpBufferWriter buffer = encoder.getBuffer();
 
                 // Copy to microbatch buffer and seal immediately
@@ -1154,12 +1150,18 @@ private void flushPendingRows() {
                 activeBuffer.incrementRowCount();
                 activeBuffer.setMaxSymbolId(currentBatchMaxSymbolId);
 
-                // Update maxSentSymbolId - once sent over TCP, server will receive it
-                maxSentSymbolId = currentBatchMaxSymbolId;
-
                 // Seal and enqueue for sending
                 sealAndSwapBuffer();
 
+                // Update sent state only after successful enqueue.
+                // If sealAndSwapBuffer() threw, these remain unchanged so the
+                // next batch's delta dictionary will correctly re-include the
+                // symbols and schema that the server never received.
+                maxSentSymbolId = currentBatchMaxSymbolId;
+                if (!useSchemaRef) {
+                    sentSchemaHashes.add(schemaKey);
+                }
+
                 // Reset table buffer and batch-level symbol tracking
                 tableBuffer.reset();
                 currentBatchMaxSymbolId = -1;
@@ -1209,11 +1211,6 @@ private void flushSync() {
                     useSchemaRef
             );
 
-            // Track schema key if this was the first time sending this schema
-            if (!useSchemaRef) {
-                sentSchemaHashes.add(schemaKey);
-            }
-
             if (messageSize > 0) {
                 QwpBufferWriter buffer = encoder.getBuffer();
 
@@ -1221,16 +1218,22 @@ private void flushSync() {
                 long batchSequence = nextBatchSequence++;
                 inFlightWindow.addInFlight(batchSequence);
 
-                // Update maxSentSymbolId - once sent over TCP, server will receive it
-                maxSentSymbolId = currentBatchMaxSymbolId;
-
-                LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef);
+                LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), currentBatchMaxSymbolId, useSchemaRef);
 
                 // Send over WebSocket
                 client.sendBinary(buffer.getBufferPtr(), messageSize);
 
                 // Wait for ACK synchronously
                 waitForAck(batchSequence);
+
+                // Update sent state only after successful send + ACK.
+                // If sendBinary() or waitForAck() threw, these remain unchanged
+                // so the next batch's delta dictionary will correctly re-include
+                // the symbols and schema that the server never received.
+                maxSentSymbolId = currentBatchMaxSymbolId;
+                if (!useSchemaRef) {
+                    sentSchemaHashes.add(schemaKey);
+                }
             }
 
             // Reset table buffer after sending
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java
new file mode 100644
index 0000000..b26f488
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java
@@ -0,0 +1,93 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.InFlightWindow;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.test.AbstractTest;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.time.temporal.ChronoUnit;
+
+/**
+ * Verifies that maxSentSymbolId and sentSchemaHashes are not updated
+ * when the send fails, so the next batch's delta dictionary correctly
+ * re-includes symbols the server never received.
+ */
+public class QwpDeltaDictRollbackTest extends AbstractTest {
+
+    @Test
+    public void testSyncFlushFailureDoesNotAdvanceMaxSentSymbolId() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            // Sync mode (window=1), not connected to any server
+            QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1);
+            try {
+                // Bypass ensureConnected() by marking as connected.
+                // Leave client null so sendBinary() will throw.
+                setField(sender, "connected", true);
+                setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS));
+
+                // Buffer a row with a symbol — this registers symbol id 0
+                // in the global dictionary and sets currentBatchMaxSymbolId = 0
+                sender.table("t")
+                        .symbol("s", "val1")
+                        .at(1, ChronoUnit.MICROS);
+
+                // maxSentSymbolId should still be -1 (nothing sent yet)
+                Assert.assertEquals(-1, sender.getMaxSentSymbolId());
+
+                // flush() -> flushSync() -> encode succeeds -> client.sendBinary() throws NPE
+                // because client is null (we never actually connected)
+                try {
+                    sender.flush();
+                    Assert.fail("Expected NullPointerException from null client");
+                } catch (NullPointerException expected) {
+                    // sendBinary() on null client
+                }
+
+                // The fix: maxSentSymbolId must remain -1 because the send failed.
+                // Without the fix, it would have been advanced to 0 before the throw,
+                // causing the next batch's delta dictionary to omit symbol "val1".
+                Assert.assertEquals(
+                        "maxSentSymbolId must not advance when send fails",
+                        -1, sender.getMaxSentSymbolId()
+                );
+            } finally {
+                // Mark as not connected so close() doesn't try to flush again
+                setField(sender, "connected", false);
+                sender.close();
+            }
+        });
+    }
+
+    private static void setField(Object target, String fieldName, Object value) throws Exception {
+        Field f = target.getClass().getDeclaredField(fieldName);
+        f.setAccessible(true);
+        f.set(target, value);
+    }
+}

From 6d7a104b2b024e313230fb0cb05ffac37e46450c Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:44:32 +0100
Subject: [PATCH 028/230] Fix TYPE_CHAR validation in QwpColumnDef

The validate() range check used TYPE_DECIMAL256 (0x15) as the upper
bound, which excluded TYPE_CHAR (0x16). CHAR columns would throw
IllegalArgumentException on validation.

Extend the upper bound to TYPE_CHAR and add tests covering all valid
type codes, nullable CHAR, and invalid type rejection.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpColumnDef.java    |  6 +-
 .../qwp/protocol/QwpColumnDefTest.java        | 97 +++++++++++++++++++
 2 files changed, 100 insertions(+), 3 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
index b9d9a26..a7257dd 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
@@ -123,9 +123,9 @@ public String getTypeName() {
      * @throws IllegalArgumentException if type code is invalid
      */
     public void validate() {
-        // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_DECIMAL256 (0x15)
-        // This includes all basic types, arrays, and decimals
-        boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_DECIMAL256);
+        // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_CHAR (0x16)
+        // This includes all basic types, arrays, decimals, and char
+        boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_CHAR);
         if (!valid) {
             throw new IllegalArgumentException(
                     "invalid column type code: 0x" + Integer.toHexString(typeCode)
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java
new file mode 100644
index 0000000..2bd28b9
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java
@@ -0,0 +1,97 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class QwpColumnDefTest {
+
+    @Test
+    public void testValidateAcceptsAllValidTypes() {
+        byte[] validTypes = {
+                QwpConstants.TYPE_BOOLEAN,
+                QwpConstants.TYPE_BYTE,
+                QwpConstants.TYPE_SHORT,
+                QwpConstants.TYPE_INT,
+                QwpConstants.TYPE_LONG,
+                QwpConstants.TYPE_FLOAT,
+                QwpConstants.TYPE_DOUBLE,
+                QwpConstants.TYPE_STRING,
+                QwpConstants.TYPE_SYMBOL,
+                QwpConstants.TYPE_TIMESTAMP,
+                QwpConstants.TYPE_DATE,
+                QwpConstants.TYPE_UUID,
+                QwpConstants.TYPE_LONG256,
+                QwpConstants.TYPE_GEOHASH,
+                QwpConstants.TYPE_VARCHAR,
+                QwpConstants.TYPE_TIMESTAMP_NANOS,
+                QwpConstants.TYPE_DOUBLE_ARRAY,
+                QwpConstants.TYPE_LONG_ARRAY,
+                QwpConstants.TYPE_DECIMAL64,
+                QwpConstants.TYPE_DECIMAL128,
+                QwpConstants.TYPE_DECIMAL256,
+                QwpConstants.TYPE_CHAR,
+        };
+        for (byte type : validTypes) {
+            QwpColumnDef col = new QwpColumnDef("col", type);
+            col.validate(); // must not throw
+        }
+    }
+
+    @Test
+    public void testValidateCharType() {
+        // TYPE_CHAR (0x16) must pass validation
+        QwpColumnDef col = new QwpColumnDef("ch", QwpConstants.TYPE_CHAR);
+        col.validate();
+        assertEquals("CHAR", col.getTypeName());
+        assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode());
+    }
+
+    @Test
+    public void testValidateNullableCharType() {
+        // TYPE_CHAR with nullable flag must also pass
+        byte nullableChar = (byte) (QwpConstants.TYPE_CHAR | QwpConstants.TYPE_NULLABLE_FLAG);
+        QwpColumnDef col = new QwpColumnDef("ch", nullableChar);
+        col.validate();
+        assertTrue(col.isNullable());
+        assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testValidateRejectsInvalidType() {
+        QwpColumnDef col = new QwpColumnDef("bad", (byte) 0x17);
+        col.validate();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testValidateRejectsZeroType() {
+        QwpColumnDef col = new QwpColumnDef("bad", (byte) 0x00);
+        col.validate();
+    }
+}

From bed5c32b126f2e26c44c8d70d9716f4bfaee0fbe Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:48:50 +0100
Subject: [PATCH 029/230] Throw LineSenderException for token in ws/wss config

Replace raw AssertionError with LineSenderException when a token
parameter is provided in ws:: or wss:: configuration strings. The
else branch in config string parsing was unreachable when the code
only supported HTTP and TCP, but became reachable after WebSocket
support was added. Users now get a clear "token is not supported
for WebSocket protocol" error instead of a cryptic AssertionError.

Add test assertions for both ws:: and wss:: schemas with token.

Co-Authored-By: Claude Opus 4.6 
---
 core/src/main/java/io/questdb/client/Sender.java                | 2 +-
 .../questdb/client/test/cutlass/line/LineSenderBuilderTest.java | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index b2b3c19..308da3a 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -1583,7 +1583,7 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
                     } else if (protocol == PROTOCOL_HTTP) {
                         httpToken(sink.toString());
                     } else {
-                        throw new AssertionError();
+                        throw new LineSenderException("token is not supported for WebSocket protocol");
                     }
                 } else if (Chars.equals("retry_timeout", sink)) {
                     pos = getValue(configurationString, pos, sink, "retry_timeout");
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java
index e684e52..0fcad48 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java
@@ -179,6 +179,8 @@ public void testConfStringValidation() throws Exception {
             assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport");
             assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset");
             assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]");
+            assertConfStrError("ws::addr=localhost;token=foo;", "token is not supported for WebSocket protocol");
+            assertConfStrError("wss::addr=localhost;token=foo;", "token is not supported for WebSocket protocol");
 
             assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1");
             assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2");

From 924e26af16eb435f220d73297aeef16830939142 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:57:16 +0100
Subject: [PATCH 030/230] Fix racy batch ID counter in MicrobatchBuffer

The static nextBatchId field was a plain long incremented with ++,
which is a non-atomic read-modify-write. Multiple threads creating
or resetting MicrobatchBuffer instances concurrently (e.g., several
Sender instances, or a user thread resetting while another constructs)
could read the same value and produce duplicate batch IDs.

Replace the plain long with an AtomicLong and use getAndIncrement()
in both the constructor and reset() to guarantee uniqueness.

Add MicrobatchBufferTest with two concurrent tests that confirm
batch ID uniqueness under contention from 8 threads.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/client/MicrobatchBuffer.java  |   7 +-
 .../qwp/client/MicrobatchBufferTest.java      | 124 ++++++++++++++++++
 2 files changed, 128 insertions(+), 3 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
index 9bcf738..4ef2af4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
@@ -30,6 +30,7 @@
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * A buffer for accumulating ILP data into microbatches before sending.
@@ -79,7 +80,7 @@ public class MicrobatchBuffer implements QuietCloseable {
 
     // Batch identification
     private long batchId;
-    private static long nextBatchId = 0;
+    private static final AtomicLong nextBatchId = new AtomicLong();
 
     // State machine
     private volatile int state = STATE_FILLING;
@@ -108,7 +109,7 @@ public MicrobatchBuffer(int initialCapacity, int maxRows, int maxBytes, long max
         this.maxRows = maxRows;
         this.maxBytes = maxBytes;
         this.maxAgeNanos = maxAgeNanos;
-        this.batchId = nextBatchId++;
+        this.batchId = nextBatchId.getAndIncrement();
     }
 
     /**
@@ -452,7 +453,7 @@ public void reset() {
         rowCount = 0;
         firstRowTimeNanos = 0;
         maxSymbolId = -1;
-        batchId = nextBatchId++;
+        batchId = nextBatchId.getAndIncrement();
         state = STATE_FILLING;
         recycleLatch = new CountDownLatch(1);
     }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java
new file mode 100644
index 0000000..cbc81ef
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java
@@ -0,0 +1,124 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer;
+import org.junit.Test;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.*;
+
+public class MicrobatchBufferTest {
+
+    @Test
+    public void testConcurrentBatchIdUniqueness() throws Exception {
+        int threadCount = 8;
+        int buffersPerThread = 500;
+        int totalBuffers = threadCount * buffersPerThread;
+        Set batchIds = ConcurrentHashMap.newKeySet();
+        CountDownLatch startLatch = new CountDownLatch(1);
+        CountDownLatch doneLatch = new CountDownLatch(threadCount);
+
+        Thread[] threads = new Thread[threadCount];
+        for (int t = 0; t < threadCount; t++) {
+            threads[t] = new Thread(() -> {
+                try {
+                    startLatch.await();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    return;
+                }
+                try {
+                    for (int i = 0; i < buffersPerThread; i++) {
+                        MicrobatchBuffer buf = new MicrobatchBuffer(64);
+                        batchIds.add(buf.getBatchId());
+                        buf.close();
+                    }
+                } finally {
+                    doneLatch.countDown();
+                }
+            });
+            threads[t].start();
+        }
+
+        startLatch.countDown();
+        assertTrue("Threads did not finish in time", doneLatch.await(30, TimeUnit.SECONDS));
+
+        assertEquals(
+                "Duplicate batch IDs detected: expected " + totalBuffers + " unique IDs but got " + batchIds.size(),
+                totalBuffers,
+                batchIds.size()
+        );
+    }
+
+    @Test
+    public void testConcurrentResetBatchIdUniqueness() throws Exception {
+        int threadCount = 8;
+        int resetsPerThread = 500;
+        int totalIds = threadCount * resetsPerThread;
+        Set batchIds = ConcurrentHashMap.newKeySet();
+        CountDownLatch startLatch = new CountDownLatch(1);
+        CountDownLatch doneLatch = new CountDownLatch(threadCount);
+
+        Thread[] threads = new Thread[threadCount];
+        for (int t = 0; t < threadCount; t++) {
+            threads[t] = new Thread(() -> {
+                try {
+                    startLatch.await();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    return;
+                }
+                try {
+                    MicrobatchBuffer buf = new MicrobatchBuffer(64);
+                    for (int i = 0; i < resetsPerThread; i++) {
+                        buf.seal();
+                        buf.markSending();
+                        buf.markRecycled();
+                        buf.reset();
+                        batchIds.add(buf.getBatchId());
+                    }
+                    buf.close();
+                } finally {
+                    doneLatch.countDown();
+                }
+            });
+            threads[t].start();
+        }
+
+        startLatch.countDown();
+        assertTrue("Threads did not finish in time", doneLatch.await(30, TimeUnit.SECONDS));
+
+        assertEquals(
+                "Duplicate batch IDs from reset(): expected " + totalIds + " unique IDs but got " + batchIds.size(),
+                totalIds,
+                batchIds.size()
+        );
+    }
+}

From 34e0cf74910cf44a360b058e4a81d408ebd17bc4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 15:16:31 +0100
Subject: [PATCH 031/230] Throw on buffer overflow in QwpBitWriter and Gorilla
 encoder

QwpBitWriter's write methods (writeBits, writeByte, writeInt,
writeLong, flush) silently dropped data when the buffer was full
instead of signaling an error. The same pattern existed in
QwpGorillaEncoder.encodeTimestamps(), which returned partial
byte counts when capacity was insufficient for the first or
second uncompressed timestamp.

Replace all silent-drop guards with LineSenderException throws
so callers get a clear error instead of silently corrupted data.
Add QwpBitWriterTest covering overflow for each write method
and both Gorilla encoder early-return paths.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpBitWriter.java    |  30 ++-
 .../qwp/protocol/QwpGorillaEncoder.java       |   5 +-
 .../qwp/protocol/QwpBitWriterTest.java        | 191 ++++++++++++++++++
 3 files changed, 213 insertions(+), 13 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
index 624b083..af30761 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.protocol;
 
+import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.std.Unsafe;
 
 /**
@@ -139,9 +140,10 @@ public void writeBits(long value, int numBits) {
 
             // Flush complete bytes from the buffer
             while (bitsInBuffer >= 8) {
-                if (currentAddress < endAddress) {
-                    Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
+                if (currentAddress >= endAddress) {
+                    throw new LineSenderException("QwpBitWriter buffer overflow");
                 }
+                Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
                 bitBuffer >>>= 8;
                 bitsInBuffer -= 8;
             }
@@ -168,7 +170,10 @@ public void writeSigned(long value, int numBits) {
      * Must be called before reading the output or getting the final position.
      */
     public void flush() {
-        if (bitsInBuffer > 0 && currentAddress < endAddress) {
+        if (bitsInBuffer > 0) {
+            if (currentAddress >= endAddress) {
+                throw new LineSenderException("QwpBitWriter buffer overflow");
+            }
             Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
             bitBuffer = 0;
             bitsInBuffer = 0;
@@ -214,9 +219,10 @@ public void alignToByte() {
      */
     public void writeByte(int value) {
         alignToByte();
-        if (currentAddress < endAddress) {
-            Unsafe.getUnsafe().putByte(currentAddress++, (byte) value);
+        if (currentAddress >= endAddress) {
+            throw new LineSenderException("QwpBitWriter buffer overflow");
         }
+        Unsafe.getUnsafe().putByte(currentAddress++, (byte) value);
     }
 
     /**
@@ -226,10 +232,11 @@ public void writeByte(int value) {
      */
     public void writeInt(int value) {
         alignToByte();
-        if (currentAddress + 4 <= endAddress) {
-            Unsafe.getUnsafe().putInt(currentAddress, value);
-            currentAddress += 4;
+        if (currentAddress + 4 > endAddress) {
+            throw new LineSenderException("QwpBitWriter buffer overflow");
         }
+        Unsafe.getUnsafe().putInt(currentAddress, value);
+        currentAddress += 4;
     }
 
     /**
@@ -239,9 +246,10 @@ public void writeInt(int value) {
      */
     public void writeLong(long value) {
         alignToByte();
-        if (currentAddress + 8 <= endAddress) {
-            Unsafe.getUnsafe().putLong(currentAddress, value);
-            currentAddress += 8;
+        if (currentAddress + 8 > endAddress) {
+            throw new LineSenderException("QwpBitWriter buffer overflow");
         }
+        Unsafe.getUnsafe().putLong(currentAddress, value);
+        currentAddress += 8;
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
index 8f59b38..912af2d 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.protocol;
 
+import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.std.Unsafe;
 
 /**
@@ -170,7 +171,7 @@ public int encodeTimestamps(long destAddress, long capacity, long srcAddress, in
 
         // Write first timestamp uncompressed
         if (capacity < 8) {
-            return 0; // Not enough space
+            throw new LineSenderException("Gorilla encoder buffer overflow");
         }
         long ts0 = Unsafe.getUnsafe().getLong(srcAddress);
         Unsafe.getUnsafe().putLong(destAddress, ts0);
@@ -182,7 +183,7 @@ public int encodeTimestamps(long destAddress, long capacity, long srcAddress, in
 
         // Write second timestamp uncompressed
         if (capacity < pos + 8) {
-            return pos; // Not enough space
+            throw new LineSenderException("Gorilla encoder buffer overflow");
         }
         long ts1 = Unsafe.getUnsafe().getLong(srcAddress + 8);
         Unsafe.getUnsafe().putLong(destAddress + pos, ts1);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java
new file mode 100644
index 0000000..11f4c36
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java
@@ -0,0 +1,191 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.protocol.QwpBitWriter;
+import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class QwpBitWriterTest {
+
+    @Test
+    public void testWriteBitsThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 4);
+            // Fill the buffer (32 bits = 4 bytes)
+            writer.writeBits(0xFFFF_FFFFL, 32);
+            // Next write should throw — buffer is full
+            try {
+                writer.writeBits(1, 8);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteByteThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 1);
+            writer.writeByte(0x42);
+            try {
+                writer.writeByte(0x43);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteIntThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 4);
+            writer.writeInt(42);
+            try {
+                writer.writeInt(99);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteLongThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 8);
+            writer.writeLong(42L);
+            try {
+                writer.writeLong(99L);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testFlushThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 1);
+            // Write 8 bits to fill the single byte
+            writer.writeBits(0xFF, 8);
+            // Write a few more bits that sit in the bit buffer
+            writer.writeBits(0x3, 4);
+            // Flush should throw because there's no room for the partial byte
+            try {
+                writer.flush();
+                fail("expected LineSenderException on buffer overflow during flush");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    // --- QwpGorillaEncoder overflow tests ---
+
+    @Test
+    public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() {
+        // Source: 1 timestamp (8 bytes), dest: only 4 bytes
+        long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS);
+        long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            Unsafe.getUnsafe().putLong(src, 1_000_000L);
+            QwpGorillaEncoder encoder = new QwpGorillaEncoder();
+            try {
+                encoder.encodeTimestamps(dst, 4, src, 1);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS);
+            Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() {
+        // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second)
+        long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS);
+        long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            Unsafe.getUnsafe().putLong(src, 1_000_000L);
+            Unsafe.getUnsafe().putLong(src + 8, 2_000_000L);
+            QwpGorillaEncoder encoder = new QwpGorillaEncoder();
+            try {
+                encoder.encodeTimestamps(dst, 12, src, 2);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS);
+            Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteBitsWithinCapacitySucceeds() {
+        long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 8);
+            writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64);
+            writer.flush();
+            assertEquals(8, writer.getPosition() - ptr);
+            assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr));
+        } finally {
+            Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+}

From ba49b6b7f53be2860d3f4f7525c7cc18cd98f16f Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 15:31:44 +0100
Subject: [PATCH 032/230] Add NativeBufferWriter tests

Add test coverage for NativeBufferWriter, which previously had no
tests. The tests exercise skip(), patchInt(), and ensureCapacity()
mechanics, including the skip-then-patch pattern used by
QwpWebSocketEncoder to reserve and backfill length fields.

skip() and patchInt() were flagged as missing bounds checks, but
investigation showed both are internal-only methods called exclusively
by QwpWebSocketEncoder with structurally guaranteed-safe arguments:
skip()'s single caller already ensures capacity before the call, and
patchInt()'s two callers always pass the hardcoded offset 8 within a
12-byte header. Adding runtime checks would be dead branches on the
hot path, so only test coverage was added.

Co-Authored-By: Claude Opus 4.6 
---
 .../qwp/client/NativeBufferWriterTest.java    | 89 +++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java
new file mode 100644
index 0000000..46eae77
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java
@@ -0,0 +1,89 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.std.Unsafe;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class NativeBufferWriterTest {
+
+    @Test
+    public void testPatchIntAtLastValidOffset() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            writer.putLong(0L); // 8 bytes, position = 8
+            // Patch at offset 4 covers bytes [4..7], exactly at the boundary
+            writer.patchInt(4, 0x1234);
+            assertEquals(0x1234, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4));
+        }
+    }
+
+    @Test
+    public void testPatchIntAtValidOffset() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            writer.putInt(0); // placeholder at offset 0
+            writer.putInt(0xBEEF); // data at offset 4
+            // Patch the placeholder
+            writer.patchInt(0, 0xCAFE);
+            assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr()));
+            assertEquals(0xBEEF, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4));
+        }
+    }
+
+    @Test
+    public void testSkipAdvancesPosition() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            writer.skip(4);
+            assertEquals(4, writer.getPosition());
+            writer.skip(8);
+            assertEquals(12, writer.getPosition());
+        }
+    }
+
+    @Test
+    public void testSkipThenPatchInt() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(8)) {
+            int patchOffset = writer.getPosition();
+            writer.skip(4); // reserve space for a length field
+            writer.putInt(0xDEAD);
+            // Patch the reserved space
+            writer.patchInt(patchOffset, 4);
+            assertEquals(0x4, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + patchOffset));
+            assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4));
+        }
+    }
+
+    @Test
+    public void testEnsureCapacityGrowsBuffer() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            assertEquals(16, writer.getCapacity());
+            writer.ensureCapacity(32);
+            assertTrue(writer.getCapacity() >= 32);
+        }
+    }
+}

From 21aa7bb03a94caded767902441416b3957f259d4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 15:46:36 +0100
Subject: [PATCH 033/230] Fix stale array offsets after cancelRow truncation

truncateTo() rewound off-heap buffers (data, strings, aux) but
forgot to rewind arrayShapeOffset and arrayDataOffset. After
cancelling a row with array data and adding a replacement row,
the new values were written at a gap past where the encoder
reads, causing the encoder to serialize stale cancelled data.

Fix by walking the retained values to recompute both offsets,
matching the same traversal pattern the encoder uses. Add tests
that verify the encoder would read correct data after cancel
for 1D double arrays, 1D long arrays, and 2D double arrays.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |  16 ++
 .../qwp/protocol/QwpTableBufferTest.java      | 216 ++++++++++++++++++
 2 files changed, 232 insertions(+)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index b2c434c..ef1a090 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -1085,6 +1085,22 @@ public void truncateTo(int newSize) {
             if (auxBuffer != null) {
                 auxBuffer.jumpTo((long) newValueCount * 4);
             }
+
+            // Rewind array offsets by walking the retained values
+            if (arrayDims != null) {
+                int newShapeOffset = 0;
+                int newDataOffset = 0;
+                for (int i = 0; i < newValueCount; i++) {
+                    int nDims = arrayDims[i];
+                    int elemCount = 1;
+                    for (int d = 0; d < nDims; d++) {
+                        elemCount *= arrayShapes[newShapeOffset++];
+                    }
+                    newDataOffset += elemCount;
+                }
+                arrayShapeOffset = newShapeOffset;
+                arrayDataOffset = newDataOffset;
+            }
         }
 
         private void allocateStorage(byte type) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
new file mode 100644
index 0000000..f11e7fe
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
@@ -0,0 +1,216 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class QwpTableBufferTest {
+
+    /**
+     * Simulates the encoder's walk over array data — the same logic as
+     * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat
+     * double values the encoder would serialize for the given column.
+     */
+    private static double[] readDoubleArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) {
+        byte[] dims = col.getArrayDims();
+        int[] shapes = col.getArrayShapes();
+        double[] data = col.getDoubleArrayData();
+        int count = col.getValueCount();
+
+        // First pass: count total elements
+        int totalElements = 0;
+        int shapeIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            totalElements += elemCount;
+        }
+
+        // Second pass: collect values
+        double[] result = new double[totalElements];
+        shapeIdx = 0;
+        int dataIdx = 0;
+        int resultIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            for (int e = 0; e < elemCount; e++) {
+                result[resultIdx++] = data[dataIdx++];
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Same as above but for long arrays (mirrors QwpWebSocketEncoder.writeLongArrayColumn()).
+     */
+    private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) {
+        byte[] dims = col.getArrayDims();
+        int[] shapes = col.getArrayShapes();
+        long[] data = col.getLongArrayData();
+        int count = col.getValueCount();
+
+        int totalElements = 0;
+        int shapeIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            totalElements += elemCount;
+        }
+
+        long[] result = new long[totalElements];
+        shapeIdx = 0;
+        int dataIdx = 0;
+        int resultIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            for (int e = 0; e < elemCount; e++) {
+                result[resultIdx++] = data[dataIdx++];
+            }
+        }
+        return result;
+    }
+
+    @Test
+    public void testCancelRowRewindsDoubleArrayOffsets() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            // Row 0: committed with [1.0, 2.0]
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{1.0, 2.0});
+            table.nextRow();
+
+            // Row 1: committed with [3.0, 4.0]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{3.0, 4.0});
+            table.nextRow();
+
+            // Start row 2 with [5.0, 6.0] — then cancel it
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{5.0, 6.0});
+            table.cancelCurrentRow();
+
+            // Add replacement row 2 with [7.0, 8.0]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{7.0, 8.0});
+            table.nextRow();
+
+            assertEquals(3, table.getRowCount());
+            assertEquals(3, col.getValueCount());
+
+            // Walk the arrays exactly as the encoder would
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 7.0, 8.0},
+                    encoded,
+                    0.0
+            );
+        }
+    }
+
+    @Test
+    public void testCancelRowRewindsLongArrayOffsets() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            // Row 0: committed with [10, 20]
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+            col.addLongArray(new long[]{10, 20});
+            table.nextRow();
+
+            // Start row 1 with [30, 40] — then cancel it
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+            col.addLongArray(new long[]{30, 40});
+            table.cancelCurrentRow();
+
+            // Add replacement row 1 with [50, 60]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+            col.addLongArray(new long[]{50, 60});
+            table.nextRow();
+
+            assertEquals(2, table.getRowCount());
+            assertEquals(2, col.getValueCount());
+
+            long[] encoded = readLongArraysLikeEncoder(col);
+            assertArrayEquals(new long[]{10, 20, 50, 60}, encoded);
+        }
+    }
+
+    @Test
+    public void testCancelRowRewindsMultiDimArrayOffsets() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            // Row 0: committed 2D array [[1.0, 2.0], [3.0, 4.0]]
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}});
+            table.nextRow();
+
+            // Start row 1 with 2D array [[5.0, 6.0], [7.0, 8.0]] — cancel
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[][]{{5.0, 6.0}, {7.0, 8.0}});
+            table.cancelCurrentRow();
+
+            // Replacement row 1 with [[9.0, 10.0], [11.0, 12.0]]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[][]{{9.0, 10.0}, {11.0, 12.0}});
+            table.nextRow();
+
+            assertEquals(2, table.getRowCount());
+            assertEquals(2, col.getValueCount());
+
+            // Verify shapes are correct (2 dims per row, each [2, 2])
+            int[] shapes = col.getArrayShapes();
+            byte[] dims = col.getArrayDims();
+            assertEquals(2, dims[0]);
+            assertEquals(2, dims[1]);
+            // Row 0 shapes: [2, 2]
+            assertEquals(2, shapes[0]);
+            assertEquals(2, shapes[1]);
+            // Row 1 shapes must be the replacement [2, 2], not stale data
+            assertEquals(2, shapes[2]);
+            assertEquals(2, shapes[3]);
+
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 9.0, 10.0, 11.0, 12.0},
+                    encoded,
+                    0.0
+            );
+        }
+    }
+}

From 1ef6d9ebd327ae37dcf5738d2cf82c7334129787 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:03:16 +0100
Subject: [PATCH 034/230] Pass auto-flush config to sync-mode WebSocket sender

The builder computed actualAutoFlushRows, actualAutoFlushBytes, and
actualAutoFlushIntervalNanos but the sync-mode WebSocket path called
QwpWebSocketSender.connect(host, port, tls), which hardcoded all
three to 0. This silently disabled auto-flush for every sync-mode
WebSocket sender, regardless of user configuration.

Add a connect() overload that accepts auto-flush parameters and
update the builder to call it. Also update createForTesting(h, p,
windowSize) to use default auto-flush values instead of zeros, so
it mirrors the production connect() path.

Co-Authored-By: Claude Opus 4.6 
---
 .../main/java/io/questdb/client/Sender.java   |  5 +++-
 .../qwp/client/QwpWebSocketSender.java        | 26 +++++++++++++++---
 .../LineSenderBuilderWebSocketTest.java       | 27 +++++++++++++++++++
 3 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 308da3a..3bd08d2 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -895,7 +895,10 @@ public Sender build() {
                     return QwpWebSocketSender.connect(
                             hosts.getQuick(0),
                             ports.getQuick(0),
-                            tlsEnabled
+                            tlsEnabled,
+                            actualAutoFlushRows,
+                            actualAutoFlushBytes,
+                            actualAutoFlushIntervalNanos
                     );
                 }
             }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index cf697d4..4deb020 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -194,7 +194,7 @@ public static QwpWebSocketSender connect(String host, int port) {
 
     /**
      * Creates a new sender with TLS and connects to the specified host and port.
-     * Uses synchronous mode for backward compatibility.
+     * Uses synchronous mode with default auto-flush settings.
      *
      * @param host       server host
      * @param port       server HTTP port
@@ -202,9 +202,28 @@ public static QwpWebSocketSender connect(String host, int port) {
      * @return connected sender
      */
     public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled) {
+        return connect(host, port, tlsEnabled,
+                DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS);
+    }
+
+    /**
+     * Creates a new sender with TLS and connects to the specified host and port.
+     * Uses synchronous mode with custom auto-flush settings.
+     *
+     * @param host                   server host
+     * @param port                   server HTTP port
+     * @param tlsEnabled             whether to use TLS
+     * @param autoFlushRows          rows per batch (0 = no limit)
+     * @param autoFlushBytes         bytes per batch (0 = no limit)
+     * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit)
+     * @return connected sender
+     */
+    public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled,
+                                              int autoFlushRows, int autoFlushBytes,
+                                              long autoFlushIntervalNanos) {
         QwpWebSocketSender sender = new QwpWebSocketSender(
                 host, port, tlsEnabled, DEFAULT_BUFFER_SIZE,
-                0, 0, 0, // No auto-flush in sync mode
+                autoFlushRows, autoFlushBytes, autoFlushIntervalNanos,
                 1, 1    // window=1 for sync behavior, queue=1 (not used)
         );
         sender.ensureConnected();
@@ -300,6 +319,7 @@ public static QwpWebSocketSender create(
      * Creates a sender without connecting. For testing only.
      * 

      * This allows unit tests to test sender logic without requiring a real server.
+     * Uses default auto-flush settings.
      *
      * @param host               server host (not connected)
      * @param port               server port (not connected)
@@ -309,7 +329,7 @@ public static QwpWebSocketSender create(
     public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) {
         return new QwpWebSocketSender(
                 host, port, false, DEFAULT_BUFFER_SIZE,
-                0, 0, 0,
+                DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS,
                 inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY
         );
         // Note: does NOT call ensureConnected()
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
index 4fb8121..4b31040 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
@@ -26,6 +26,7 @@
 
 import io.questdb.client.Sender;
 import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
 import io.questdb.client.test.AbstractTest;
 import io.questdb.client.test.tools.TestUtils;
 import org.junit.Assert;
@@ -621,6 +622,32 @@ public void testSyncModeDoesNotAllowSendQueueCapacity() {
 
     // ==================== Unsupported Features (HTTP-specific options) ====================
 
+    @Test
+    public void testSyncModeAutoFlushDefaults() throws Exception {
+        // Regression test: sync-mode connect() must not hardcode autoFlush to 0.
+        // createForTesting(host, port, windowSize) mirrors what connect(h,p,tls)
+        // creates internally. Verify it uses sensible defaults.
+        TestUtils.assertMemoryLeak(() -> {
+            QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1);
+            try {
+                Assert.assertEquals(
+                        QwpWebSocketSender.DEFAULT_AUTO_FLUSH_ROWS,
+                        sender.getAutoFlushRows()
+                );
+                Assert.assertEquals(
+                        QwpWebSocketSender.DEFAULT_AUTO_FLUSH_BYTES,
+                        sender.getAutoFlushBytes()
+                );
+                Assert.assertEquals(
+                        QwpWebSocketSender.DEFAULT_AUTO_FLUSH_INTERVAL_NANOS,
+                        sender.getAutoFlushIntervalNanos()
+                );
+            } finally {
+                sender.close();
+            }
+        });
+    }
+
     @Test
     public void testSyncModeIsDefault() {
         Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET)

From 38d96c76b0c0bf2f148a90b2730a8331aca9850e Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:10:52 +0100
Subject: [PATCH 035/230] Add DECIMAL64/128/256 to isFixedWidthType and
 getFixedTypeSize

These decimal types are fixed-width (8, 16, and 32 bytes respectively)
but were missing from both isFixedWidthType() and getFixedTypeSize()
in QwpConstants. This caused them to be incorrectly classified as
variable-width types.

Co-Authored-By: Claude Opus 4.6 
---
 .../questdb/client/cutlass/qwp/protocol/QwpConstants.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
index 622dbdd..2a40a35 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
@@ -384,7 +384,10 @@ public static boolean isFixedWidthType(byte typeCode) {
                 code == TYPE_TIMESTAMP_NANOS ||
                 code == TYPE_DATE ||
                 code == TYPE_UUID ||
-                code == TYPE_LONG256;
+                code == TYPE_LONG256 ||
+                code == TYPE_DECIMAL64 ||
+                code == TYPE_DECIMAL128 ||
+                code == TYPE_DECIMAL256;
     }
 
     /**
@@ -411,10 +414,13 @@ public static int getFixedTypeSize(byte typeCode) {
             case TYPE_TIMESTAMP:
             case TYPE_TIMESTAMP_NANOS:
             case TYPE_DATE:
+            case TYPE_DECIMAL64:
                 return 8;
             case TYPE_UUID:
+            case TYPE_DECIMAL128:
                 return 16;
             case TYPE_LONG256:
+            case TYPE_DECIMAL256:
                 return 32;
             default:
                 return -1; // Variable width

From e65c882f7332306f385930b546a289095d92eca0 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:29:12 +0100
Subject: [PATCH 036/230] Pool ArrayCapture to eliminate per-row allocations

Make ArrayCapture a reusable instance field of ColumnBuffer instead
of allocating a new one (plus int[32], double[N], long[N]) on every
addDoubleArray(DoubleArray) / addLongArray(LongArray) call. The
pooled instance is reset() between uses; its internal data arrays
grow but never shrink, so repeated same-shaped rows allocate nothing
after the first call.

Also fix two pre-existing issues in ArrayCapture:
- putBlockOfBytes() always read native memory as doubles via
  getDouble(), so addLongArray(LongArray) would copy zero data
  elements (longDataOffset was never incremented). Now dispatches
  on a forLong flag to use getLong() for long arrays.
- putInt() unconditionally allocated both doubleData and longData
  arrays even though only one type is ever used per call. Now
  allocates only the needed array type.

Add tests covering the DoubleArray wrapper path (multiple rows,
shrinking sizes, varying dimensionality) and multi-row long array
data isolation.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |  77 +++++---
 .../qwp/protocol/QwpTableBufferTest.java      | 164 ++++++++++++++++++
 2 files changed, 216 insertions(+), 25 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index ef1a090..b56cd28 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -296,20 +296,38 @@ static int elementSize(byte type) {
     private static class ArrayCapture implements ArrayBufferAppender {
         double[] doubleData;
         int doubleDataOffset;
+        private boolean forLong;
         long[] longData;
         int longDataOffset;
         byte nDims;
-        int[] shape = new int[32];
-        int shapeIndex;
+        final int[] shape = new int[32];
+        private int shapeIndex;
+
+        void reset(boolean forLong) {
+            this.forLong = forLong;
+            shapeIndex = 0;
+            nDims = 0;
+            doubleDataOffset = 0;
+            longDataOffset = 0;
+        }
 
         @Override
         public void putBlockOfBytes(long from, long len) {
             int count = (int) (len / 8);
-            if (doubleData == null) {
-                doubleData = new double[count];
-            }
-            for (int i = 0; i < count; i++) {
-                doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
+            if (forLong) {
+                if (longData == null || longData.length < count) {
+                    longData = new long[count];
+                }
+                for (int i = 0; i < count; i++) {
+                    longData[longDataOffset++] = Unsafe.getUnsafe().getLong(from + i * 8L);
+                }
+            } else {
+                if (doubleData == null || doubleData.length < count) {
+                    doubleData = new double[count];
+                }
+                for (int i = 0; i < count; i++) {
+                    doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
+                }
             }
         }
 
@@ -336,8 +354,15 @@ public void putInt(int value) {
                     for (int i = 0; i < nDims; i++) {
                         totalElements *= shape[i];
                     }
-                    doubleData = new double[totalElements];
-                    longData = new long[totalElements];
+                    if (forLong) {
+                        if (longData == null || longData.length < totalElements) {
+                            longData = new long[totalElements];
+                        }
+                    } else {
+                        if (doubleData == null || doubleData.length < totalElements) {
+                            doubleData = new double[totalElements];
+                        }
+                    }
                 }
             }
         }
@@ -362,6 +387,7 @@ public static class ColumnBuffer implements QuietCloseable {
         final boolean nullable;
         final byte type;
         private final Decimal256 rescaleTemp = new Decimal256();
+        private ArrayCapture arrayCapture;
         private int arrayDataOffset;
         // Array storage (double/long arrays - variable length per row)
         private byte[] arrayDims;
@@ -573,16 +599,16 @@ public void addDoubleArray(DoubleArray array) {
                 addNull();
                 return;
             }
-            ArrayCapture capture = new ArrayCapture();
-            array.appendToBufPtr(capture);
+            arrayCapture.reset(false);
+            array.appendToBufPtr(arrayCapture);
 
-            ensureArrayCapacity(capture.nDims, capture.doubleDataOffset);
-            arrayDims[valueCount] = capture.nDims;
-            for (int i = 0; i < capture.nDims; i++) {
-                arrayShapes[arrayShapeOffset++] = capture.shape[i];
+            ensureArrayCapacity(arrayCapture.nDims, arrayCapture.doubleDataOffset);
+            arrayDims[valueCount] = arrayCapture.nDims;
+            for (int i = 0; i < arrayCapture.nDims; i++) {
+                arrayShapes[arrayShapeOffset++] = arrayCapture.shape[i];
             }
-            for (int i = 0; i < capture.doubleDataOffset; i++) {
-                doubleArrayData[arrayDataOffset++] = capture.doubleData[i];
+            for (int i = 0; i < arrayCapture.doubleDataOffset; i++) {
+                doubleArrayData[arrayDataOffset++] = arrayCapture.doubleData[i];
             }
             valueCount++;
             size++;
@@ -698,16 +724,16 @@ public void addLongArray(LongArray array) {
                 addNull();
                 return;
             }
-            ArrayCapture capture = new ArrayCapture();
-            array.appendToBufPtr(capture);
+            arrayCapture.reset(true);
+            array.appendToBufPtr(arrayCapture);
 
-            ensureArrayCapacity(capture.nDims, capture.longDataOffset);
-            arrayDims[valueCount] = capture.nDims;
-            for (int i = 0; i < capture.nDims; i++) {
-                arrayShapes[arrayShapeOffset++] = capture.shape[i];
+            ensureArrayCapacity(arrayCapture.nDims, arrayCapture.longDataOffset);
+            arrayDims[valueCount] = arrayCapture.nDims;
+            for (int i = 0; i < arrayCapture.nDims; i++) {
+                arrayShapes[arrayShapeOffset++] = arrayCapture.shape[i];
             }
-            for (int i = 0; i < capture.longDataOffset; i++) {
-                longArrayData[arrayDataOffset++] = capture.longData[i];
+            for (int i = 0; i < arrayCapture.longDataOffset; i++) {
+                longArrayData[arrayDataOffset++] = arrayCapture.longData[i];
             }
             valueCount++;
             size++;
@@ -1148,6 +1174,7 @@ private void allocateStorage(byte type) {
                 case TYPE_DOUBLE_ARRAY:
                 case TYPE_LONG_ARRAY:
                     arrayDims = new byte[16];
+                    arrayCapture = new ArrayCapture();
                     break;
                 case TYPE_DECIMAL64:
                     dataBuffer = new OffHeapAppendMemory(128);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
index f11e7fe..501893e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.test.cutlass.qwp.protocol;
 
+import io.questdb.client.cutlass.line.array.DoubleArray;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
 import org.junit.Test;
@@ -213,4 +214,167 @@ public void testCancelRowRewindsMultiDimArrayOffsets() {
             );
         }
     }
+
+    @Test
+    public void testDoubleArrayWrapperMultipleRows() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test");
+             DoubleArray arr = new DoubleArray(3)) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+
+            arr.append(1.0).append(2.0).append(3.0);
+            col.addDoubleArray(arr);
+            table.nextRow();
+
+            // DoubleArray auto-wraps, so just append next row's data
+            arr.append(4.0).append(5.0).append(6.0);
+            col.addDoubleArray(arr);
+            table.nextRow();
+
+            arr.append(7.0).append(8.0).append(9.0);
+            col.addDoubleArray(arr);
+            table.nextRow();
+
+            assertEquals(3, col.getValueCount());
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0},
+                    encoded,
+                    0.0
+            );
+
+            byte[] dims = col.getArrayDims();
+            int[] shapes = col.getArrayShapes();
+            for (int i = 0; i < 3; i++) {
+                assertEquals(1, dims[i]);
+                assertEquals(3, shapes[i]);
+            }
+        }
+    }
+
+    @Test
+    public void testDoubleArrayWrapperShrinkingSize() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+
+            // Row 0: large array (5 elements)
+            try (DoubleArray big = new DoubleArray(5)) {
+                big.append(1.0).append(2.0).append(3.0).append(4.0).append(5.0);
+                col.addDoubleArray(big);
+                table.nextRow();
+            }
+
+            // Row 1: smaller array (2 elements) — must not see leftover data from row 0
+            try (DoubleArray small = new DoubleArray(2)) {
+                small.append(10.0).append(20.0);
+                col.addDoubleArray(small);
+                table.nextRow();
+            }
+
+            assertEquals(2, col.getValueCount());
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0},
+                    encoded,
+                    0.0
+            );
+
+            int[] shapes = col.getArrayShapes();
+            assertEquals(5, shapes[0]);
+            assertEquals(2, shapes[1]);
+        }
+    }
+
+    @Test
+    public void testDoubleArrayWrapperVaryingDimensionality() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+
+            // Row 0: 2D array (2x2)
+            try (DoubleArray matrix = new DoubleArray(2, 2)) {
+                matrix.append(1.0).append(2.0).append(3.0).append(4.0);
+                col.addDoubleArray(matrix);
+                table.nextRow();
+            }
+
+            // Row 1: 1D array (3 elements) — different dimensionality
+            try (DoubleArray vec = new DoubleArray(3)) {
+                vec.append(10.0).append(20.0).append(30.0);
+                col.addDoubleArray(vec);
+                table.nextRow();
+            }
+
+            assertEquals(2, col.getValueCount());
+
+            byte[] dims = col.getArrayDims();
+            assertEquals(2, dims[0]);
+            assertEquals(1, dims[1]);
+
+            int[] shapes = col.getArrayShapes();
+            // Row 0: shape [2, 2]
+            assertEquals(2, shapes[0]);
+            assertEquals(2, shapes[1]);
+            // Row 1: shape [3]
+            assertEquals(3, shapes[2]);
+
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 30.0},
+                    encoded,
+                    0.0
+            );
+        }
+    }
+
+    @Test
+    public void testLongArrayMultipleRows() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+
+            col.addLongArray(new long[]{10, 20, 30});
+            table.nextRow();
+
+            col.addLongArray(new long[]{40, 50, 60});
+            table.nextRow();
+
+            col.addLongArray(new long[]{70, 80, 90});
+            table.nextRow();
+
+            assertEquals(3, col.getValueCount());
+            long[] encoded = readLongArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90},
+                    encoded
+            );
+
+            byte[] dims = col.getArrayDims();
+            int[] shapes = col.getArrayShapes();
+            for (int i = 0; i < 3; i++) {
+                assertEquals(1, dims[i]);
+                assertEquals(3, shapes[i]);
+            }
+        }
+    }
+
+    @Test
+    public void testLongArrayShrinkingSize() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+
+            // Row 0: large array (4 elements)
+            col.addLongArray(new long[]{100, 200, 300, 400});
+            table.nextRow();
+
+            // Row 1: smaller array (2 elements) — must not see leftover data from row 0
+            col.addLongArray(new long[]{10, 20});
+            table.nextRow();
+
+            assertEquals(2, col.getValueCount());
+            long[] encoded = readLongArraysLikeEncoder(col);
+            assertArrayEquals(new long[]{100, 200, 300, 400, 10, 20}, encoded);
+
+            int[] shapes = col.getArrayShapes();
+            assertEquals(4, shapes[0]);
+            assertEquals(2, shapes[1]);
+        }
+    }
 }

From 4c0c2fff679ecdbea019494f1d54a80ce8a53964 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:33:59 +0100
Subject: [PATCH 037/230] Remove dead parseBuffer allocation in ResponseReader

The parseBufferPtr field (2KB native memory via Unsafe.malloc) was
allocated in the constructor and freed in close(), but never actually
read from or written to. The readLoop() receives response data
through channel.receiveFrame() which passes the channel's own buffer
pointer directly to ResponseHandlerImpl.onBinaryMessage(), bypassing
parseBufferPtr entirely.

Remove the unused fields, allocation, deallocation, and the
now-unnecessary Unsafe and MemoryTag imports.

Co-Authored-By: Claude Opus 4.6 
---
 .../client/cutlass/qwp/client/ResponseReader.java | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java
index 2b78429..1d2b89f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java
@@ -27,9 +27,7 @@
 import io.questdb.client.cutlass.line.LineSenderException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
-import io.questdb.client.std.Unsafe;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -61,10 +59,6 @@ public class ResponseReader implements QuietCloseable {
     private final CountDownLatch shutdownLatch;
     private final WebSocketResponse response;
 
-    // Buffer for parsing responses
-    private final long parseBufferPtr;
-    private final int parseBufferSize;
-
     // State
     private volatile boolean running;
     private volatile Throwable lastError;
@@ -91,10 +85,6 @@ public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) {
         this.inFlightWindow = inFlightWindow;
         this.response = new WebSocketResponse();
 
-        // Allocate parse buffer (enough for max response)
-        this.parseBufferSize = 2048;
-        this.parseBufferPtr = Unsafe.malloc(parseBufferSize, MemoryTag.NATIVE_DEFAULT);
-
         this.running = true;
         this.shutdownLatch = new CountDownLatch(1);
 
@@ -151,11 +141,6 @@ public void close() {
             Thread.currentThread().interrupt();
         }
 
-        // Free parse buffer
-        if (parseBufferPtr != 0) {
-            Unsafe.free(parseBufferPtr, parseBufferSize, MemoryTag.NATIVE_DEFAULT);
-        }
-
         LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get());
     }
 

From f71a1eed9876295dd123939356904775349a1364 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:37:29 +0100
Subject: [PATCH 038/230] Auto-clean up QwpBitReader

---
 .../cutlass/qwp/protocol/QwpBitReader.java    | 211 +++++++++---------
 1 file changed, 104 insertions(+), 107 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java
index bd99092..9241d70 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java
@@ -45,14 +45,12 @@
  */
 public class QwpBitReader {
 
-    private long startAddress;
-    private long currentAddress;
-    private long endAddress;
-
     // Buffer for reading bits
     private long bitBuffer;
     // Number of bits currently available in the buffer (0-64)
     private int bitsInBuffer;
+    private long currentAddress;
+    private long endAddress;
     // Total bits available for reading (from reset)
     private long totalBitsAvailable;
     // Total bits already consumed
@@ -65,75 +63,73 @@ public QwpBitReader() {
     }
 
     /**
-     * Resets the reader to read from the specified memory region.
+     * Aligns the reader to the next byte boundary by skipping any partial bits.
      *
-     * @param address the starting address
-     * @param length  the number of bytes available to read
+     * @throws IllegalStateException if alignment fails
      */
-    public void reset(long address, long length) {
-        this.startAddress = address;
-        this.currentAddress = address;
-        this.endAddress = address + length;
-        this.bitBuffer = 0;
-        this.bitsInBuffer = 0;
-        this.totalBitsAvailable = length * 8L;
-        this.totalBitsRead = 0;
+    public void alignToByte() {
+        int bitsToSkip = bitsInBuffer % 8;
+        if (bitsToSkip != 0) {
+            // We need to skip the remaining bits in the current partial byte
+            // But since we read in byte chunks, bitsInBuffer should be a multiple of 8
+            // minus what we've consumed. The remainder in the conceptual stream is:
+            int remainder = (int) (totalBitsRead % 8);
+            if (remainder != 0) {
+                skipBits(8 - remainder);
+            }
+        }
     }
 
     /**
-     * Resets the reader to read from the specified byte array.
+     * Returns the number of bits remaining to be read.
      *
-     * @param buf    the byte array
-     * @param offset the starting offset
-     * @param length the number of bytes available
+     * @return available bits
      */
-    public void reset(byte[] buf, int offset, int length) {
-        // For byte array, we'll store position info differently
-        // This is mainly for testing - in production we use direct memory
-        throw new UnsupportedOperationException("Use direct memory version");
+    public long getAvailableBits() {
+        return totalBitsAvailable - totalBitsRead;
     }
 
     /**
-     * Returns the number of bits remaining to be read.
+     * Returns the current position in bits from the start.
      *
-     * @return available bits
+     * @return bits read since reset
      */
-    public long getAvailableBits() {
-        return totalBitsAvailable - totalBitsRead;
+    public long getBitPosition() {
+        return totalBitsRead;
     }
 
     /**
-     * Returns true if there are more bits to read.
+     * Returns the current byte address being read.
+     * Note: This is approximate due to bit buffering.
      *
-     * @return true if bits available
+     * @return current address
      */
-    public boolean hasMoreBits() {
-        return totalBitsRead < totalBitsAvailable;
+    public long getCurrentAddress() {
+        return currentAddress;
     }
 
     /**
-     * Returns the current position in bits from the start.
+     * Returns true if there are more bits to read.
      *
-     * @return bits read since reset
+     * @return true if bits available
      */
-    public long getBitPosition() {
-        return totalBitsRead;
+    public boolean hasMoreBits() {
+        return totalBitsRead < totalBitsAvailable;
     }
 
     /**
-     * Ensures the buffer has at least the requested number of bits.
-     * Loads more bytes from memory if needed.
+     * Peeks at the next bit without consuming it.
      *
-     * @param bitsNeeded minimum bits required in buffer
-     * @return true if sufficient bits available, false otherwise
+     * @return 0 or 1, or -1 if no more bits
      */
-    private boolean ensureBits(int bitsNeeded) {
-        while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) {
-            byte b = Unsafe.getUnsafe().getByte(currentAddress++);
-            bitBuffer |= (long) (b & 0xFF) << bitsInBuffer;
-            bitsInBuffer += 8;
+    public int peekBit() {
+        if (totalBitsRead >= totalBitsAvailable) {
+            return -1;
         }
-        return bitsInBuffer >= bitsNeeded;
+        if (!ensureBits(1)) {
+            return -1;
+        }
+        return (int) (bitBuffer & 1);
     }
 
     /**
@@ -203,6 +199,36 @@ public long readBits(int numBits) {
         return result;
     }
 
+    /**
+     * Reads a complete byte, ensuring byte alignment first.
+     *
+     * @return the byte value (0-255)
+     * @throws IllegalStateException if not enough data
+     */
+    public int readByte() {
+        return (int) readBits(8) & 0xFF;
+    }
+
+    /**
+     * Reads a complete 32-bit integer in little-endian order.
+     *
+     * @return the integer value
+     * @throws IllegalStateException if not enough data
+     */
+    public int readInt() {
+        return (int) readBits(32);
+    }
+
+    /**
+     * Reads a complete 64-bit long in little-endian order.
+     *
+     * @return the long value
+     * @throws IllegalStateException if not enough data
+     */
+    public long readLong() {
+        return readBits(64);
+    }
+
     /**
      * Reads multiple bits and interprets them as a signed value using two's complement.
      *
@@ -221,18 +247,31 @@ public long readSigned(int numBits) {
     }
 
     /**
-     * Peeks at the next bit without consuming it.
+     * Resets the reader to read from the specified memory region.
      *
-     * @return 0 or 1, or -1 if no more bits
+     * @param address the starting address
+     * @param length  the number of bytes available to read
      */
-    public int peekBit() {
-        if (totalBitsRead >= totalBitsAvailable) {
-            return -1;
-        }
-        if (!ensureBits(1)) {
-            return -1;
-        }
-        return (int) (bitBuffer & 1);
+    public void reset(long address, long length) {
+        this.currentAddress = address;
+        this.endAddress = address + length;
+        this.bitBuffer = 0;
+        this.bitsInBuffer = 0;
+        this.totalBitsAvailable = length * 8L;
+        this.totalBitsRead = 0;
+    }
+
+    /**
+     * Resets the reader to read from the specified byte array.
+     *
+     * @param buf    the byte array
+     * @param offset the starting offset
+     * @param length the number of bytes available
+     */
+    public void reset(byte[] buf, int offset, int length) {
+        // For byte array, we'll store position info differently
+        // This is mainly for testing - in production we use direct memory
+        throw new UnsupportedOperationException("Use direct memory version");
     }
 
     /**
@@ -276,60 +315,18 @@ public void skipBits(int numBits) {
     }
 
     /**
-     * Aligns the reader to the next byte boundary by skipping any partial bits.
+     * Ensures the buffer has at least the requested number of bits.
+     * Loads more bytes from memory if needed.
      *
-     * @throws IllegalStateException if alignment fails
+     * @param bitsNeeded minimum bits required in buffer
+     * @return true if sufficient bits available, false otherwise
      */
-    public void alignToByte() {
-        int bitsToSkip = bitsInBuffer % 8;
-        if (bitsToSkip != 0) {
-            // We need to skip the remaining bits in the current partial byte
-            // But since we read in byte chunks, bitsInBuffer should be a multiple of 8
-            // minus what we've consumed. The remainder in the conceptual stream is:
-            int remainder = (int) (totalBitsRead % 8);
-            if (remainder != 0) {
-                skipBits(8 - remainder);
-            }
+    private boolean ensureBits(int bitsNeeded) {
+        while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) {
+            byte b = Unsafe.getUnsafe().getByte(currentAddress++);
+            bitBuffer |= (long) (b & 0xFF) << bitsInBuffer;
+            bitsInBuffer += 8;
         }
-    }
-
-    /**
-     * Reads a complete byte, ensuring byte alignment first.
-     *
-     * @return the byte value (0-255)
-     * @throws IllegalStateException if not enough data
-     */
-    public int readByte() {
-        return (int) readBits(8) & 0xFF;
-    }
-
-    /**
-     * Reads a complete 32-bit integer in little-endian order.
-     *
-     * @return the integer value
-     * @throws IllegalStateException if not enough data
-     */
-    public int readInt() {
-        return (int) readBits(32);
-    }
-
-    /**
-     * Reads a complete 64-bit long in little-endian order.
-     *
-     * @return the long value
-     * @throws IllegalStateException if not enough data
-     */
-    public long readLong() {
-        return readBits(64);
-    }
-
-    /**
-     * Returns the current byte address being read.
-     * Note: This is approximate due to bit buffering.
-     *
-     * @return current address
-     */
-    public long getCurrentAddress() {
-        return currentAddress;
+        return bitsInBuffer >= bitsNeeded;
     }
 }

From b85d8903abcb82f26d84b558775ed67dd625e1e4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Wed, 25 Feb 2026 09:28:59 +0100
Subject: [PATCH 039/230] Sort members alphabetically, remove section headings

Reorder class members (fields, methods) alphabetically by kind
and visibility across all java-questdb-client source and test
files. This follows the project convention of alphabetical
member ordering.

Remove decorative section-heading comments (// ===, // ---,
// ====================) that no longer serve a purpose after
alphabetical reordering. Incorporate the orphaned "Fast-path
API" comment block into the QwpWebSocketSender class Javadoc.

Co-Authored-By: Claude Opus 4.6 
---
 .../main/java/io/questdb/client/Sender.java   |  120 +-
 .../cutlass/http/client/WebSocketClient.java  |  899 +--
 .../http/client/WebSocketFrameHandler.java    |   24 +-
 .../http/client/WebSocketSendBuffer.java      |  492 +-
 .../client/cutlass/json/JsonLexer.java        |    3 +-
 .../cutlass/line/AbstractLineSender.java      |    4 +-
 .../qwp/client/GlobalSymbolDictionary.java    |   98 +-
 .../cutlass/qwp/client/InFlightWindow.java    |  337 +-
 .../cutlass/qwp/client/MicrobatchBuffer.java  |  409 +-
 .../qwp/client/NativeBufferWriter.java        |  192 +-
 .../cutlass/qwp/client/QwpBufferWriter.java   |  148 +-
 .../qwp/client/QwpWebSocketSender.java        |   67 +-
 .../cutlass/qwp/client/ResponseReader.java    |   68 +-
 .../cutlass/qwp/client/WebSocketChannel.java  |  460 +-
 .../cutlass/qwp/client/WebSocketResponse.java |  146 +-
 .../qwp/client/WebSocketSendQueue.java        |  364 +-
 .../qwp/protocol/OffHeapAppendMemory.java     |   85 +-
 .../cutlass/qwp/protocol/QwpBitWriter.java    |  149 +-
 .../cutlass/qwp/protocol/QwpColumnDef.java    |   91 +-
 .../cutlass/qwp/protocol/QwpConstants.java    |  378 +-
 .../qwp/protocol/QwpGorillaEncoder.java       |  156 +-
 .../cutlass/qwp/protocol/QwpNullBitmap.java   |  212 +-
 .../cutlass/qwp/protocol/QwpSchemaHash.java   |  506 +-
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |   20 +-
 .../cutlass/qwp/protocol/QwpVarint.java       |  126 +-
 .../cutlass/qwp/protocol/QwpZigZag.java       |   24 +-
 .../qwp/websocket/WebSocketCloseCode.java     |  118 +-
 .../qwp/websocket/WebSocketFrameParser.java   |  175 +-
 .../qwp/websocket/WebSocketFrameWriter.java   |  260 +-
 .../qwp/websocket/WebSocketHandshake.java     |  350 +-
 .../qwp/websocket/WebSocketOpcode.java        |   27 +-
 .../java/io/questdb/client/network/Net.java   |   26 +-
 .../client/std/CharSequenceIntHashMap.java    |   46 +-
 .../client/std/bytes/DirectByteSink.java      |    3 +-
 .../line/tcp/v4/QwpAllocationTestClient.java  |  148 +-
 .../line/tcp/v4/StacBenchmarkClient.java      |  201 +-
 .../qwp/client/InFlightWindowTest.java        |  872 +-
 .../LineSenderBuilderWebSocketTest.java       |   94 +-
 .../qwp/client/MicrobatchBufferTest.java      |    3 +-
 .../qwp/client/NativeBufferWriterTest.java    |   18 +-
 .../cutlass/qwp/client/QwpSenderTest.java     | 7112 ++++++++---------
 .../qwp/client/WebSocketChannelTest.java      |  197 +-
 .../qwp/protocol/OffHeapAppendMemoryTest.java |  303 +-
 .../qwp/protocol/QwpBitWriterTest.java        |  136 +-
 .../qwp/protocol/QwpColumnDefTest.java        |    3 +-
 .../qwp/protocol/QwpTableBufferTest.java      |  159 +-
 46 files changed, 7771 insertions(+), 8058 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 3bd08d2..c998b78 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -531,12 +531,18 @@ final class LineSenderBuilder {
         private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024;
         private static final int DEFAULT_HTTP_PORT = 9000;
         private static final int DEFAULT_HTTP_TIMEOUT = 30_000;
+        private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8;
         private static final int DEFAULT_MAXIMUM_BUFFER_CAPACITY = 100 * 1024 * 1024;
         private static final int DEFAULT_MAX_BACKOFF_MILLIS = 1_000;
         private static final int DEFAULT_MAX_NAME_LEN = 127;
         private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method
         private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method
+        private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16;
         private static final int DEFAULT_TCP_PORT = 9009;
+        private static final int DEFAULT_WEBSOCKET_PORT = 9000;
+        private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB
+        private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms
+        private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500;
         private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1;
         // The PARAMETER_NOT_SET_EXPLICITLY constant is used to detect if a parameter was set explicitly in configuration parameters
         // where it matters. This is needed to detect invalid combinations of parameters. Why?
@@ -546,14 +552,10 @@ final class LineSenderBuilder {
         private static final int PROTOCOL_HTTP = 1;
         private static final int PROTOCOL_TCP = 0;
         private static final int PROTOCOL_WEBSOCKET = 2;
-        private static final int DEFAULT_WEBSOCKET_PORT = 9000;
-        private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8;
-        private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16;
-        private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500;
-        private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB
-        private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms
         private final ObjList hosts = new ObjList<>();
         private final IntList ports = new IntList();
+        private boolean asyncMode = false;
+        private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY;
         private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY;
         private int autoFlushRows = PARAMETER_NOT_SET_EXPLICITLY;
         private int bufferCapacity = PARAMETER_NOT_SET_EXPLICITLY;
@@ -561,6 +563,8 @@ final class LineSenderBuilder {
         private String httpSettingsPath;
         private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY;
         private String httpToken;
+        // WebSocket-specific fields
+        private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY;
         private String keyId;
         private int maxBackoffMillis = PARAMETER_NOT_SET_EXPLICITLY;
         private int maxNameLength = PARAMETER_NOT_SET_EXPLICITLY;
@@ -592,17 +596,13 @@ public int getTimeout() {
         private int protocol = PARAMETER_NOT_SET_EXPLICITLY;
         private int protocolVersion = PARAMETER_NOT_SET_EXPLICITLY;
         private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY;
+        private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY;
         private boolean shouldDestroyPrivKey;
         private boolean tlsEnabled;
         private TlsValidationMode tlsValidationMode;
         private char[] trustStorePassword;
         private String trustStorePath;
         private String username;
-        // WebSocket-specific fields
-        private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY;
-        private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY;
-        private boolean asyncMode = false;
-        private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY;
 
         private LineSenderBuilder() {
 
@@ -693,6 +693,47 @@ public AdvancedTlsSettings advancedTls() {
             return new AdvancedTlsSettings();
         }
 
+        /**
+         * Enable asynchronous mode for WebSocket transport.
+         * 

+         * In async mode, rows are batched and sent asynchronously with flow control.
+         * This provides higher throughput at the cost of more complex error handling.
+         * 

+         * This is only used when communicating over WebSocket transport.
+         * 

+         * Default is synchronous mode (false).
+         *
+         * @param enabled whether to enable async mode
+         * @return this instance for method chaining
+         */
+        public LineSenderBuilder asyncMode(boolean enabled) {
+            this.asyncMode = enabled;
+            return this;
+        }
+
+        /**
+         * Set the maximum number of bytes per batch before auto-flushing.
+         * 

+         * This is only used when communicating over WebSocket transport.
+         * 

+         * Default value is 1MB.
+         *
+         * @param bytes maximum bytes per batch
+         * @return this instance for method chaining
+         */
+        public LineSenderBuilder autoFlushBytes(int bytes) {
+            if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) {
+                throw new LineSenderException("auto flush bytes was already configured")
+                        .put("[bytes=").put(this.autoFlushBytes).put("]");
+            }
+            if (bytes < 0) {
+                throw new LineSenderException("auto flush bytes cannot be negative")
+                        .put("[bytes=").put(bytes).put("]");
+            }
+            this.autoFlushBytes = bytes;
+            return this;
+        }
+
         /**
          * Set the interval in milliseconds at which the Sender automatically flushes its buffer.
          * 

@@ -768,47 +809,6 @@ public LineSenderBuilder autoFlushRows(int autoFlushRows) {
             return this;
         }
 
-        /**
-         * Set the maximum number of bytes per batch before auto-flushing.
-         * 

-         * This is only used when communicating over WebSocket transport.
-         * 

-         * Default value is 1MB.
-         *
-         * @param bytes maximum bytes per batch
-         * @return this instance for method chaining
-         */
-        public LineSenderBuilder autoFlushBytes(int bytes) {
-            if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) {
-                throw new LineSenderException("auto flush bytes was already configured")
-                        .put("[bytes=").put(this.autoFlushBytes).put("]");
-            }
-            if (bytes < 0) {
-                throw new LineSenderException("auto flush bytes cannot be negative")
-                        .put("[bytes=").put(bytes).put("]");
-            }
-            this.autoFlushBytes = bytes;
-            return this;
-        }
-
-        /**
-         * Enable asynchronous mode for WebSocket transport.
-         * 

-         * In async mode, rows are batched and sent asynchronously with flow control.
-         * This provides higher throughput at the cost of more complex error handling.
-         * 

-         * This is only used when communicating over WebSocket transport.
-         * 

-         * Default is synchronous mode (false).
-         *
-         * @param enabled whether to enable async mode
-         * @return this instance for method chaining
-         */
-        public LineSenderBuilder asyncMode(boolean enabled) {
-            this.asyncMode = enabled;
-            return this;
-        }
-
         /**
          * Configure capacity of an internal buffer.
          * 

@@ -1729,14 +1729,6 @@ private void tcp() {
             protocol = PROTOCOL_TCP;
         }
 
-        private void websocket() {
-            if (protocol != PARAMETER_NOT_SET_EXPLICITLY) {
-                throw new LineSenderException("protocol was already configured ")
-                        .put("[protocol=").put(protocol).put("]");
-            }
-            protocol = PROTOCOL_WEBSOCKET;
-        }
-
         private void validateParameters() {
             if (hosts.size() == 0) {
                 throw new LineSenderException("questdb server address not set");
@@ -1817,6 +1809,14 @@ private void validateParameters() {
             }
         }
 
+        private void websocket() {
+            if (protocol != PARAMETER_NOT_SET_EXPLICITLY) {
+                throw new LineSenderException("protocol was already configured ")
+                        .put("[protocol=").put(protocol).put("]");
+            }
+            protocol = PROTOCOL_WEBSOCKET;
+        }
+
         public class AdvancedTlsSettings {
             /**
              * Configure a custom truststore. This is only needed when using {@link #enableTls()} when your default
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index fec618f..a0cf1a3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -67,41 +67,34 @@
  */
 public abstract class WebSocketClient implements QuietCloseable {
 
-    private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class);
-
     private static final int DEFAULT_RECV_BUFFER_SIZE = 65536;
     private static final int DEFAULT_SEND_BUFFER_SIZE = 65536;
-
+    private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class);
     protected final NetworkFacade nf;
     protected final Socket socket;
-
-    private final WebSocketSendBuffer sendBuffer;
     private final WebSocketSendBuffer controlFrameBuffer;
-    private final WebSocketFrameParser frameParser;
-    private final Rnd rnd;
     private final int defaultTimeout;
+    private final WebSocketFrameParser frameParser;
     private final int maxRecvBufSize;
-
+    private final Rnd rnd;
+    private final WebSocketSendBuffer sendBuffer;
+    private boolean closed;
+    private int fragmentBufPos;
+    private long fragmentBufPtr;       // native buffer for accumulating fragment payloads
+    private int fragmentBufSize;
+    // Fragmentation state (RFC 6455 Section 5.4)
+    private int fragmentOpcode = -1;   // opcode of first fragment, -1 = not in a fragmented message
+    // Handshake key for verification
+    private String handshakeKey;
+    // Connection state
+    private CharSequence host;
+    private int port;
     // Receive buffer (native memory)
     private long recvBufPtr;
     private int recvBufSize;
     private int recvPos;      // Write position
     private int recvReadPos;  // Read position
-
-    // Connection state
-    private CharSequence host;
-    private int port;
     private boolean upgraded;
-    private boolean closed;
-
-    // Fragmentation state (RFC 6455 Section 5.4)
-    private int fragmentOpcode = -1;   // opcode of first fragment, -1 = not in a fragmented message
-    private long fragmentBufPtr;       // native buffer for accumulating fragment payloads
-    private int fragmentBufSize;
-    private int fragmentBufPos;
-
-    // Handshake key for verification
-    private String handshakeKey;
 
     public WebSocketClient(HttpClientConfiguration configuration, SocketFactory socketFactory) {
         this.nf = configuration.getNetworkFacade();
@@ -158,20 +151,6 @@ public void close() {
         }
     }
 
-    /**
-     * Disconnects the socket without closing the client.
-     * The client can be reconnected by calling connect() again.
-     */
-    public void disconnect() {
-        Misc.free(socket);
-        upgraded = false;
-        host = null;
-        port = 0;
-        recvPos = 0;
-        recvReadPos = 0;
-        resetFragmentState();
-    }
-
     /**
      * Connects to a WebSocket server.
      *
@@ -204,206 +183,34 @@ public void connect(CharSequence host, int port) {
         connect(host, port, defaultTimeout);
     }
 
-    private void doConnect(CharSequence host, int port, int timeout) {
-        int fd = nf.socketTcp(true);
-        if (fd < 0) {
-            throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']');
-        }
-
-        if (nf.setTcpNoDelay(fd, true) < 0) {
-            LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno());
-        }
-
-        socket.of(fd);
-        nf.configureKeepAlive(fd);
-
-        long addrInfo = nf.getAddrInfo(host, port);
-        if (addrInfo == -1) {
-            disconnect();
-            throw new HttpClientException("could not resolve host [host=").put(host).put(']');
-        }
-
-        if (nf.connectAddrInfo(fd, addrInfo) != 0) {
-            int errno = nf.errno();
-            nf.freeAddrInfo(addrInfo);
-            disconnect();
-            throw new HttpClientException("could not connect [host=").put(host)
-                    .put(", port=").put(port)
-                    .put(", errno=").put(errno).put(']');
-        }
-        nf.freeAddrInfo(addrInfo);
-
-        if (nf.configureNonBlocking(fd) < 0) {
-            int errno = nf.errno();
-            disconnect();
-            throw new HttpClientException("could not configure non-blocking [fd=").put(fd)
-                    .put(", errno=").put(errno).put(']');
-        }
-
-        if (socket.supportsTls()) {
-            try {
-                socket.startTlsSession(host);
-            } catch (TlsSessionInitFailedException e) {
-                int errno = nf.errno();
-                disconnect();
-                throw new HttpClientException("could not start TLS session [fd=").put(fd)
-                        .put(", error=").put(e.getFlyweightMessage())
-                        .put(", errno=").put(errno).put(']');
-            }
-        }
-
-        setupIoWait();
-        LOG.debug("Connected to [host={}, port={}]", host, port);
-    }
-
     /**
-     * Performs WebSocket upgrade handshake.
-     *
-     * @param path    the WebSocket endpoint path (e.g., "/ws")
-     * @param timeout timeout in milliseconds
+     * Disconnects the socket without closing the client.
+     * The client can be reconnected by calling connect() again.
      */
-    public void upgrade(CharSequence path, int timeout) {
-        if (closed) {
-            throw new HttpClientException("WebSocket client is closed");
-        }
-        if (socket.isClosed()) {
-            throw new HttpClientException("Not connected");
-        }
-        if (upgraded) {
-            return; // Already upgraded
-        }
-
-        // Generate random key
-        byte[] keyBytes = new byte[16];
-        for (int i = 0; i < 16; i++) {
-            keyBytes[i] = (byte) rnd.nextInt(256);
-        }
-        handshakeKey = Base64.getEncoder().encodeToString(keyBytes);
-
-        // Build upgrade request
-        sendBuffer.reset();
-        sendBuffer.putAscii("GET ");
-        sendBuffer.putAscii(path);
-        sendBuffer.putAscii(" HTTP/1.1\r\n");
-        sendBuffer.putAscii("Host: ");
-        sendBuffer.putAscii(host);
-        if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) {
-            sendBuffer.putAscii(":");
-            sendBuffer.putAscii(Integer.toString(port));
-        }
-        sendBuffer.putAscii("\r\n");
-        sendBuffer.putAscii("Upgrade: websocket\r\n");
-        sendBuffer.putAscii("Connection: Upgrade\r\n");
-        sendBuffer.putAscii("Sec-WebSocket-Key: ");
-        sendBuffer.putAscii(handshakeKey);
-        sendBuffer.putAscii("\r\n");
-        sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n");
-        sendBuffer.putAscii("\r\n");
-
-        // Send request
-        long startTime = System.nanoTime();
-        doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout);
-
-        // Read response
-        int remainingTimeout = remainingTime(timeout, startTime);
-        readUpgradeResponse(remainingTimeout);
-
-        upgraded = true;
-        sendBuffer.reset();
-        LOG.debug("WebSocket upgraded [path={}]", path);
+    public void disconnect() {
+        Misc.free(socket);
+        upgraded = false;
+        host = null;
+        port = 0;
+        recvPos = 0;
+        recvReadPos = 0;
+        resetFragmentState();
     }
 
     /**
-     * Performs upgrade with default timeout.
+     * Returns the connected host.
      */
-    public void upgrade(CharSequence path) {
-        upgrade(path, defaultTimeout);
-    }
-
-    private void readUpgradeResponse(int timeout) {
-        // Read HTTP response into receive buffer
-        long startTime = System.nanoTime();
-
-        while (true) {
-            int remainingTimeout = remainingTime(timeout, startTime);
-            int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
-            if (bytesRead > 0) {
-                recvPos += bytesRead;
-            }
-
-            // Check for end of headers (\r\n\r\n)
-            int headerEnd = findHeaderEnd();
-            if (headerEnd > 0) {
-                validateUpgradeResponse(headerEnd);
-                // Compact buffer - move remaining data to start
-                int remaining = recvPos - headerEnd;
-                if (remaining > 0) {
-                    Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining);
-                }
-                recvPos = remaining;
-                recvReadPos = 0;
-                return;
-            }
-
-            if (recvPos >= recvBufSize) {
-                throw new HttpClientException("HTTP response too large");
-            }
-        }
-    }
-
-    private int findHeaderEnd() {
-        // Look for \r\n\r\n
-        for (int i = 0; i < recvPos - 3; i++) {
-            if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' &&
-                Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' &&
-                Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' &&
-                Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') {
-                return i + 4;
-            }
-        }
-        return -1;
-    }
-
-    private void validateUpgradeResponse(int headerEnd) {
-        // Extract response as string for parsing
-        byte[] responseBytes = new byte[headerEnd];
-        for (int i = 0; i < headerEnd; i++) {
-            responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i);
-        }
-        String response = new String(responseBytes, StandardCharsets.US_ASCII);
-
-        // Check status line
-        if (!response.startsWith("HTTP/1.1 101")) {
-            String statusLine = response.split("\r\n")[0];
-            throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine);
-        }
-
-        // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230)
-        String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey);
-        if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) {
-            throw new HttpClientException("Invalid Sec-WebSocket-Accept header");
-        }
+    public CharSequence getHost() {
+        return host;
     }
 
-    private static boolean containsHeaderValue(String response, String headerName, String expectedValue) {
-        int headerLen = headerName.length();
-        int responseLen = response.length();
-        for (int i = 0; i <= responseLen - headerLen; i++) {
-            if (response.regionMatches(true, i, headerName, 0, headerLen)) {
-                int valueStart = i + headerLen;
-                int lineEnd = response.indexOf('\r', valueStart);
-                if (lineEnd < 0) {
-                    lineEnd = responseLen;
-                }
-                String actualValue = response.substring(valueStart, lineEnd).trim();
-                return actualValue.equals(expectedValue);
-            }
-        }
-        return false;
+    /**
+     * Returns the connected port.
+     */
+    public int getPort() {
+        return port;
     }
 
-    // === Sending ===
-
     /**
      * Gets the send buffer for building WebSocket frames.
      * 

@@ -422,21 +229,59 @@ public WebSocketSendBuffer getSendBuffer() {
     }
 
     /**
-     * Sends a complete WebSocket frame.
+     * Returns whether the WebSocket is connected and upgraded.
+     */
+    public boolean isConnected() {
+        return upgraded && !closed && !socket.isClosed();
+    }
+
+    /**
+     * Receives and processes WebSocket frames.
      *
-     * @param frame   frame info from endBinaryFrame()
+     * @param handler frame handler callback
      * @param timeout timeout in milliseconds
+     * @return true if a frame was received, false on timeout
      */
-    public void sendFrame(WebSocketSendBuffer.FrameInfo frame, int timeout) {
+    public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) {
         checkConnected();
-        doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+
+        // First, try to parse any data already in buffer
+        Boolean result = tryParseFrame(handler);
+        if (result != null) {
+            return result;
+        }
+
+        // Need more data
+        long startTime = System.nanoTime();
+        while (true) {
+            int remainingTimeout = remainingTime(timeout, startTime);
+            if (remainingTimeout <= 0) {
+                return false; // Timeout
+            }
+
+            // Ensure buffer has space
+            if (recvPos >= recvBufSize - 1024) {
+                growRecvBuffer();
+            }
+
+            int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
+            if (bytesRead <= 0) {
+                return false; // Timeout
+            }
+            recvPos += bytesRead;
+
+            result = tryParseFrame(handler);
+            if (result != null) {
+                return result;
+            }
+        }
     }
 
     /**
-     * Sends a complete WebSocket frame with default timeout.
+     * Receives frame with default timeout.
      */
-    public void sendFrame(WebSocketSendBuffer.FrameInfo frame) {
-        sendFrame(frame, defaultTimeout);
+    public boolean receiveFrame(WebSocketFrameHandler handler) {
+        return receiveFrame(handler, defaultTimeout);
     }
 
     /**
@@ -463,6 +308,37 @@ public void sendBinary(long dataPtr, int length) {
         sendBinary(dataPtr, length, defaultTimeout);
     }
 
+    /**
+     * Sends a close frame.
+     */
+    public void sendCloseFrame(int code, String reason, int timeout) {
+        sendBuffer.reset();
+        WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason);
+        try {
+            doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+        } finally {
+            sendBuffer.reset();
+        }
+    }
+
+    /**
+     * Sends a complete WebSocket frame.
+     *
+     * @param frame   frame info from endBinaryFrame()
+     * @param timeout timeout in milliseconds
+     */
+    public void sendFrame(WebSocketSendBuffer.FrameInfo frame, int timeout) {
+        checkConnected();
+        doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+    }
+
+    /**
+     * Sends a complete WebSocket frame with default timeout.
+     */
+    public void sendFrame(WebSocketSendBuffer.FrameInfo frame) {
+        sendFrame(frame, defaultTimeout);
+    }
+
     /**
      * Sends a ping frame.
      */
@@ -475,16 +351,222 @@ public void sendPing(int timeout) {
     }
 
     /**
-     * Sends a close frame.
+     * Non-blocking attempt to receive a WebSocket frame.
+     * Returns immediately if no complete frame is available.
+     *
+     * @param handler frame handler callback
+     * @return true if a frame was received, false if no data available
      */
-    public void sendCloseFrame(int code, String reason, int timeout) {
+    public boolean tryReceiveFrame(WebSocketFrameHandler handler) {
+        checkConnected();
+
+        // First, try to parse any data already in buffer
+        Boolean result = tryParseFrame(handler);
+        if (result != null) {
+            return result;
+        }
+
+        // Try one non-blocking recv
+        if (recvPos >= recvBufSize - 1024) {
+            growRecvBuffer();
+        }
+
+        int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos);
+        if (n < 0) {
+            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+        }
+        if (n == 0) {
+            return false; // No data available
+        }
+        recvPos += n;
+
+        // Try to parse again
+        result = tryParseFrame(handler);
+        return result != null && result;
+    }
+
+    /**
+     * Performs WebSocket upgrade handshake.
+     *
+     * @param path    the WebSocket endpoint path (e.g., "/ws")
+     * @param timeout timeout in milliseconds
+     */
+    public void upgrade(CharSequence path, int timeout) {
+        if (closed) {
+            throw new HttpClientException("WebSocket client is closed");
+        }
+        if (socket.isClosed()) {
+            throw new HttpClientException("Not connected");
+        }
+        if (upgraded) {
+            return; // Already upgraded
+        }
+
+        // Generate random key
+        byte[] keyBytes = new byte[16];
+        for (int i = 0; i < 16; i++) {
+            keyBytes[i] = (byte) rnd.nextInt(256);
+        }
+        handshakeKey = Base64.getEncoder().encodeToString(keyBytes);
+
+        // Build upgrade request
         sendBuffer.reset();
-        WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason);
-        try {
-            doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
-        } finally {
-            sendBuffer.reset();
+        sendBuffer.putAscii("GET ");
+        sendBuffer.putAscii(path);
+        sendBuffer.putAscii(" HTTP/1.1\r\n");
+        sendBuffer.putAscii("Host: ");
+        sendBuffer.putAscii(host);
+        if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) {
+            sendBuffer.putAscii(":");
+            sendBuffer.putAscii(Integer.toString(port));
+        }
+        sendBuffer.putAscii("\r\n");
+        sendBuffer.putAscii("Upgrade: websocket\r\n");
+        sendBuffer.putAscii("Connection: Upgrade\r\n");
+        sendBuffer.putAscii("Sec-WebSocket-Key: ");
+        sendBuffer.putAscii(handshakeKey);
+        sendBuffer.putAscii("\r\n");
+        sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n");
+        sendBuffer.putAscii("\r\n");
+
+        // Send request
+        long startTime = System.nanoTime();
+        doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout);
+
+        // Read response
+        int remainingTimeout = remainingTime(timeout, startTime);
+        readUpgradeResponse(remainingTimeout);
+
+        upgraded = true;
+        sendBuffer.reset();
+        LOG.debug("WebSocket upgraded [path={}]", path);
+    }
+
+    /**
+     * Performs upgrade with default timeout.
+     */
+    public void upgrade(CharSequence path) {
+        upgrade(path, defaultTimeout);
+    }
+
+    private static boolean containsHeaderValue(String response, String headerName, String expectedValue) {
+        int headerLen = headerName.length();
+        int responseLen = response.length();
+        for (int i = 0; i <= responseLen - headerLen; i++) {
+            if (response.regionMatches(true, i, headerName, 0, headerLen)) {
+                int valueStart = i + headerLen;
+                int lineEnd = response.indexOf('\r', valueStart);
+                if (lineEnd < 0) {
+                    lineEnd = responseLen;
+                }
+                String actualValue = response.substring(valueStart, lineEnd).trim();
+                return actualValue.equals(expectedValue);
+            }
+        }
+        return false;
+    }
+
+    private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
+        if (payloadLen == 0) {
+            return;
+        }
+        int required = fragmentBufPos + payloadLen;
+        if (required > maxRecvBufSize) {
+            throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=")
+                    .put(required)
+                    .put(", max=")
+                    .put(maxRecvBufSize)
+                    .put(']');
+        }
+        if (fragmentBufPtr == 0) {
+            fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
+            fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
+        } else if (required > fragmentBufSize) {
+            int newSize = Math.min(Math.max(fragmentBufSize * 2, required), maxRecvBufSize);
+            fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+            fragmentBufSize = newSize;
+        }
+        Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen);
+        fragmentBufPos += payloadLen;
+    }
+
+    private void checkConnected() {
+        if (closed) {
+            throw new HttpClientException("WebSocket client is closed");
+        }
+        if (!upgraded) {
+            throw new HttpClientException("WebSocket not connected or upgraded");
+        }
+    }
+
+    private void compactRecvBuffer() {
+        if (recvReadPos > 0) {
+            int remaining = recvPos - recvReadPos;
+            if (remaining > 0) {
+                Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining);
+            }
+            recvPos = remaining;
+            recvReadPos = 0;
+        }
+    }
+
+    private int dieIfNegative(int byteCount) {
+        if (byteCount < 0) {
+            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
         }
+        return byteCount;
+    }
+
+    private void doConnect(CharSequence host, int port, int timeout) {
+        int fd = nf.socketTcp(true);
+        if (fd < 0) {
+            throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']');
+        }
+
+        if (nf.setTcpNoDelay(fd, true) < 0) {
+            LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno());
+        }
+
+        socket.of(fd);
+        nf.configureKeepAlive(fd);
+
+        long addrInfo = nf.getAddrInfo(host, port);
+        if (addrInfo == -1) {
+            disconnect();
+            throw new HttpClientException("could not resolve host [host=").put(host).put(']');
+        }
+
+        if (nf.connectAddrInfo(fd, addrInfo) != 0) {
+            int errno = nf.errno();
+            nf.freeAddrInfo(addrInfo);
+            disconnect();
+            throw new HttpClientException("could not connect [host=").put(host)
+                    .put(", port=").put(port)
+                    .put(", errno=").put(errno).put(']');
+        }
+        nf.freeAddrInfo(addrInfo);
+
+        if (nf.configureNonBlocking(fd) < 0) {
+            int errno = nf.errno();
+            disconnect();
+            throw new HttpClientException("could not configure non-blocking [fd=").put(fd)
+                    .put(", errno=").put(errno).put(']');
+        }
+
+        if (socket.supportsTls()) {
+            try {
+                socket.startTlsSession(host);
+            } catch (TlsSessionInitFailedException e) {
+                int errno = nf.errno();
+                disconnect();
+                throw new HttpClientException("could not start TLS session [fd=").put(fd)
+                        .put(", error=").put(e.getFlyweightMessage())
+                        .put(", errno=").put(errno).put(']');
+            }
+        }
+
+        setupIoWait();
+        LOG.debug("Connected to [host={}, port={}]", host, port);
     }
 
     private void doSend(long ptr, int len, int timeout) {
@@ -505,90 +587,130 @@ private void doSend(long ptr, int len, int timeout) {
         }
     }
 
-    // === Receiving ===
-
-    /**
-     * Receives and processes WebSocket frames.
-     *
-     * @param handler frame handler callback
-     * @param timeout timeout in milliseconds
-     * @return true if a frame was received, false on timeout
-     */
-    public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) {
-        checkConnected();
+    private int findHeaderEnd() {
+        // Look for \r\n\r\n
+        for (int i = 0; i < recvPos - 3; i++) {
+            if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' &&
+                    Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' &&
+                    Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' &&
+                    Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') {
+                return i + 4;
+            }
+        }
+        return -1;
+    }
 
-        // First, try to parse any data already in buffer
-        Boolean result = tryParseFrame(handler);
-        if (result != null) {
-            return result;
+    private void growRecvBuffer() {
+        int newSize = recvBufSize * 2;
+        if (newSize > maxRecvBufSize) {
+            if (recvBufSize >= maxRecvBufSize) {
+                throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=")
+                        .put(recvBufSize)
+                        .put(", max=")
+                        .put(maxRecvBufSize)
+                        .put(']');
+            }
+            newSize = maxRecvBufSize;
         }
+        recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+        recvBufSize = newSize;
+    }
 
-        // Need more data
+    private void readUpgradeResponse(int timeout) {
+        // Read HTTP response into receive buffer
         long startTime = System.nanoTime();
+
         while (true) {
             int remainingTimeout = remainingTime(timeout, startTime);
-            if (remainingTimeout <= 0) {
-                return false; // Timeout
-            }
-
-            // Ensure buffer has space
-            if (recvPos >= recvBufSize - 1024) {
-                growRecvBuffer();
-            }
-
-            int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
-            if (bytesRead <= 0) {
-                return false; // Timeout
+            int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
+            if (bytesRead > 0) {
+                recvPos += bytesRead;
             }
-            recvPos += bytesRead;
 
-            result = tryParseFrame(handler);
-            if (result != null) {
-                return result;
+            // Check for end of headers (\r\n\r\n)
+            int headerEnd = findHeaderEnd();
+            if (headerEnd > 0) {
+                validateUpgradeResponse(headerEnd);
+                // Compact buffer - move remaining data to start
+                int remaining = recvPos - headerEnd;
+                if (remaining > 0) {
+                    Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining);
+                }
+                recvPos = remaining;
+                recvReadPos = 0;
+                return;
             }
-        }
-    }
-
-    /**
-     * Receives frame with default timeout.
-     */
-    public boolean receiveFrame(WebSocketFrameHandler handler) {
-        return receiveFrame(handler, defaultTimeout);
-    }
-
-    /**
-     * Non-blocking attempt to receive a WebSocket frame.
-     * Returns immediately if no complete frame is available.
-     *
-     * @param handler frame handler callback
-     * @return true if a frame was received, false if no data available
-     */
-    public boolean tryReceiveFrame(WebSocketFrameHandler handler) {
-        checkConnected();
 
-        // First, try to parse any data already in buffer
-        Boolean result = tryParseFrame(handler);
-        if (result != null) {
-            return result;
+            if (recvPos >= recvBufSize) {
+                throw new HttpClientException("HTTP response too large");
+            }
         }
+    }
 
-        // Try one non-blocking recv
-        if (recvPos >= recvBufSize - 1024) {
-            growRecvBuffer();
+    private int recvOrDie(long ptr, int len, int timeout) {
+        long startTime = System.nanoTime();
+        int n = dieIfNegative(socket.recv(ptr, len));
+        if (n == 0) {
+            ioWait(remainingTime(timeout, startTime), IOOperation.READ);
+            n = dieIfNegative(socket.recv(ptr, len));
         }
+        return n;
+    }
 
-        int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos);
+    private int recvOrTimeout(long ptr, int len, int timeout) {
+        long startTime = System.nanoTime();
+        int n = socket.recv(ptr, len);
         if (n < 0) {
             throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
         }
         if (n == 0) {
-            return false; // No data available
+            try {
+                ioWait(timeout, IOOperation.READ);
+            } catch (HttpClientException e) {
+                // Timeout
+                return 0;
+            }
+            n = socket.recv(ptr, len);
+            if (n < 0) {
+                throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+            }
         }
-        recvPos += n;
+        return n;
+    }
 
-        // Try to parse again
-        result = tryParseFrame(handler);
-        return result != null && result;
+    private int remainingTime(int timeoutMillis, long startTimeNanos) {
+        timeoutMillis -= (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos);
+        if (timeoutMillis <= 0) {
+            throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']');
+        }
+        return timeoutMillis;
+    }
+
+    private void resetFragmentState() {
+        fragmentOpcode = -1;
+        fragmentBufPos = 0;
+    }
+
+    private void sendCloseFrameEcho(int code) {
+        try {
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
+        } catch (Exception e) {
+            LOG.error("Failed to echo close frame: {}", e.getMessage());
+        }
+    }
+
+    private void sendPongFrame(long payloadPtr, int payloadLen) {
+        try {
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
+        } catch (Exception e) {
+            LOG.error("Failed to send pong: {}", e.getMessage());
+        }
     }
 
     private Boolean tryParseFrame(WebSocketFrameHandler handler) {
@@ -600,7 +722,7 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
         int consumed = frameParser.parse(recvBufPtr + recvReadPos, recvBufPtr + recvPos);
 
         if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE ||
-            frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) {
+                frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) {
             return null; // Need more data
         }
 
@@ -706,130 +828,25 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
         return false;
     }
 
-    private void sendCloseFrameEcho(int code) {
-        try {
-            controlFrameBuffer.reset();
-            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null);
-            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
-            controlFrameBuffer.reset();
-        } catch (Exception e) {
-            LOG.error("Failed to echo close frame: {}", e.getMessage());
-        }
-    }
-
-    private void sendPongFrame(long payloadPtr, int payloadLen) {
-        try {
-            controlFrameBuffer.reset();
-            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen);
-            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
-            controlFrameBuffer.reset();
-        } catch (Exception e) {
-            LOG.error("Failed to send pong: {}", e.getMessage());
-        }
-    }
-
-    private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
-        if (payloadLen == 0) {
-            return;
-        }
-        int required = fragmentBufPos + payloadLen;
-        if (required > maxRecvBufSize) {
-            throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=")
-                    .put(required)
-                    .put(", max=")
-                    .put(maxRecvBufSize)
-                    .put(']');
-        }
-        if (fragmentBufPtr == 0) {
-            fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
-            fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
-        } else if (required > fragmentBufSize) {
-            int newSize = Math.min(Math.max(fragmentBufSize * 2, required), maxRecvBufSize);
-            fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
-            fragmentBufSize = newSize;
-        }
-        Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen);
-        fragmentBufPos += payloadLen;
-    }
-
-    private void resetFragmentState() {
-        fragmentOpcode = -1;
-        fragmentBufPos = 0;
-    }
-
-    private void compactRecvBuffer() {
-        if (recvReadPos > 0) {
-            int remaining = recvPos - recvReadPos;
-            if (remaining > 0) {
-                Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining);
-            }
-            recvPos = remaining;
-            recvReadPos = 0;
-        }
-    }
-
-    private void growRecvBuffer() {
-        int newSize = recvBufSize * 2;
-        if (newSize > maxRecvBufSize) {
-            if (recvBufSize >= maxRecvBufSize) {
-                throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=")
-                        .put(recvBufSize)
-                        .put(", max=")
-                        .put(maxRecvBufSize)
-                        .put(']');
-            }
-            newSize = maxRecvBufSize;
-        }
-        recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
-        recvBufSize = newSize;
-    }
-
-    // === Socket I/O helpers ===
-
-    private int recvOrDie(long ptr, int len, int timeout) {
-        long startTime = System.nanoTime();
-        int n = dieIfNegative(socket.recv(ptr, len));
-        if (n == 0) {
-            ioWait(remainingTime(timeout, startTime), IOOperation.READ);
-            n = dieIfNegative(socket.recv(ptr, len));
-        }
-        return n;
-    }
-
-    private int recvOrTimeout(long ptr, int len, int timeout) {
-        long startTime = System.nanoTime();
-        int n = socket.recv(ptr, len);
-        if (n < 0) {
-            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
-        }
-        if (n == 0) {
-            try {
-                ioWait(timeout, IOOperation.READ);
-            } catch (HttpClientException e) {
-                // Timeout
-                return 0;
-            }
-            n = socket.recv(ptr, len);
-            if (n < 0) {
-                throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
-            }
+    private void validateUpgradeResponse(int headerEnd) {
+        // Extract response as string for parsing
+        byte[] responseBytes = new byte[headerEnd];
+        for (int i = 0; i < headerEnd; i++) {
+            responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i);
         }
-        return n;
-    }
+        String response = new String(responseBytes, StandardCharsets.US_ASCII);
 
-    private int dieIfNegative(int byteCount) {
-        if (byteCount < 0) {
-            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+        // Check status line
+        if (!response.startsWith("HTTP/1.1 101")) {
+            String statusLine = response.split("\r\n")[0];
+            throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine);
         }
-        return byteCount;
-    }
 
-    private int remainingTime(int timeoutMillis, long startTimeNanos) {
-        timeoutMillis -= (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos);
-        if (timeoutMillis <= 0) {
-            throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']');
+        // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230)
+        String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey);
+        if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) {
+            throw new HttpClientException("Invalid Sec-WebSocket-Accept header");
         }
-        return timeoutMillis;
     }
 
     protected void dieWaiting(int n) {
@@ -842,40 +859,6 @@ protected void dieWaiting(int n) {
         throw new HttpClientException("queue error [errno=").put(nf.errno()).put(']');
     }
 
-    private void checkConnected() {
-        if (closed) {
-            throw new HttpClientException("WebSocket client is closed");
-        }
-        if (!upgraded) {
-            throw new HttpClientException("WebSocket not connected or upgraded");
-        }
-    }
-
-    // === State ===
-
-    /**
-     * Returns whether the WebSocket is connected and upgraded.
-     */
-    public boolean isConnected() {
-        return upgraded && !closed && !socket.isClosed();
-    }
-
-    /**
-     * Returns the connected host.
-     */
-    public CharSequence getHost() {
-        return host;
-    }
-
-    /**
-     * Returns the connected port.
-     */
-    public int getPort() {
-        return port;
-    }
-
-    // === Platform-specific I/O ===
-
     /**
      * Waits for I/O readiness using platform-specific mechanism.
      *
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
index aff429d..e3682f5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
@@ -43,18 +43,6 @@ public interface WebSocketFrameHandler {
      */
     void onBinaryMessage(long payloadPtr, int payloadLen);
 
-    /**
-     * Called when a text frame is received.
-     * 

-     * Default implementation does nothing. Override if text frames need handling.
-     *
-     * @param payloadPtr pointer to the UTF-8 encoded payload in native memory
-     * @param payloadLen length of the payload in bytes
-     */
-    default void onTextMessage(long payloadPtr, int payloadLen) {
-        // Default: ignore text frames
-    }
-
     /**
      * Called when a close frame is received from the server.
      * 

@@ -90,4 +78,16 @@ default void onPing(long payloadPtr, int payloadLen) {
     default void onPong(long payloadPtr, int payloadLen) {
         // Default: ignore pong frames
     }
+
+    /**
+     * Called when a text frame is received.
+     * 

+     * Default implementation does nothing. Override if text frames need handling.
+     *
+     * @param payloadPtr pointer to the UTF-8 encoded payload in native memory
+     * @param payloadLen length of the payload in bytes
+     */
+    default void onTextMessage(long payloadPtr, int payloadLen) {
+        // Default: ignore text frames
+    }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
index 5075b9f..861de15 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
@@ -24,10 +24,10 @@
 
 package io.questdb.client.cutlass.http.client;
 
+import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
+import io.questdb.client.cutlass.qwp.client.QwpBufferWriter;
 import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter;
 import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode;
-import io.questdb.client.cutlass.qwp.client.QwpBufferWriter;
-import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.Numbers;
 import io.questdb.client.std.QuietCloseable;
@@ -57,21 +57,18 @@
  */
 public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable {
 
-    // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key)
-    private static final int MAX_HEADER_SIZE = 14;
-
     private static final int DEFAULT_INITIAL_CAPACITY = 65536;
     private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; // Leave room for alignment
-
-    private long bufPtr;
+    // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key)
+    private static final int MAX_HEADER_SIZE = 14;
+    private final FrameInfo frameInfo = new FrameInfo();
+    private final int maxBufferSize;
+    private final Rnd rnd;
     private int bufCapacity;
-    private int writePos;           // Current write position (offset from bufPtr)
+    private long bufPtr;
     private int frameStartOffset;   // Where current frame's reserved header starts
     private int payloadStartOffset; // Where payload begins (frameStart + MAX_HEADER_SIZE)
-
-    private final Rnd rnd;
-    private final int maxBufferSize;
-    private final FrameInfo frameInfo = new FrameInfo();
+    private int writePos;           // Current write position (offset from bufPtr)
 
     /**
      * Creates a new WebSocket send buffer with default initial capacity.
@@ -105,6 +102,34 @@ public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) {
         this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis());
     }
 
+    /**
+     * Begins a new binary WebSocket frame. Reserves space for the maximum header size.
+     * After calling this method, use ArrayBufferAppender methods to write the payload.
+     */
+    public void beginBinaryFrame() {
+        beginFrame(WebSocketOpcode.BINARY);
+    }
+
+    /**
+     * Begins a new WebSocket frame with the specified opcode.
+     *
+     * @param opcode the frame opcode
+     */
+    public void beginFrame(int opcode) {
+        frameStartOffset = writePos;
+        // Reserve maximum header space
+        ensureCapacity(MAX_HEADER_SIZE);
+        writePos += MAX_HEADER_SIZE;
+        payloadStartOffset = writePos;
+    }
+
+    /**
+     * Begins a new text WebSocket frame. Reserves space for the maximum header size.
+     */
+    public void beginTextFrame() {
+        beginFrame(WebSocketOpcode.TEXT);
+    }
+
     @Override
     public void close() {
         if (bufPtr != 0) {
@@ -114,7 +139,53 @@ public void close() {
         }
     }
 
-    // === Buffer Management ===
+    /**
+     * Finishes the current binary frame, writing the header and applying masking.
+     * Returns information about where to find the complete frame in the buffer.
+     * 

+     * IMPORTANT: Only call this after all payload writes are complete. The buffer
+     * pointer is stable after this call (no more reallocations for this frame).
+     *
+     * @return frame info containing offset and length for sending
+     */
+    public FrameInfo endBinaryFrame() {
+        return endFrame(WebSocketOpcode.BINARY);
+    }
+
+    /**
+     * Finishes the current frame with the specified opcode.
+     *
+     * @param opcode the frame opcode
+     * @return frame info containing offset and length for sending
+     */
+    public FrameInfo endFrame(int opcode) {
+        int payloadLen = writePos - payloadStartOffset;
+
+        // Calculate actual header size (with mask key for client frames)
+        int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+        int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize;
+        int actualFrameStart = frameStartOffset + unusedSpace;
+
+        // Generate mask key
+        int maskKey = rnd.nextInt();
+
+        // Write header at actual position (after unused space)
+        WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey);
+
+        // Apply mask to payload
+        if (payloadLen > 0) {
+            WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey);
+        }
+
+        return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen);
+    }
+
+    /**
+     * Finishes the current text frame, writing the header and applying masking.
+     */
+    public FrameInfo endTextFrame() {
+        return endFrame(WebSocketOpcode.TEXT);
+    }
 
     /**
      * Ensures the buffer has capacity for the specified number of additional bytes.
@@ -130,50 +201,63 @@ public void ensureCapacity(int additionalBytes) {
         }
     }
 
-    private void grow(long requiredCapacity) {
-        if (requiredCapacity > maxBufferSize) {
-            throw new HttpClientException("WebSocket buffer size exceeded maximum [required=")
-                    .put(requiredCapacity)
-                    .put(", max=")
-                    .put(maxBufferSize)
-                    .put(']');
-        }
-        int newCapacity = Math.min(
-                Numbers.ceilPow2((int) requiredCapacity),
-                maxBufferSize
-        );
-        bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
-        bufCapacity = newCapacity;
+    /**
+     * Gets the buffer pointer. Only use this for reading after frame is complete.
+     */
+    public long getBufferPtr() {
+        return bufPtr;
     }
 
-    // === ArrayBufferAppender Implementation ===
+    /**
+     * Gets the current buffer capacity.
+     */
+    public int getCapacity() {
+        return bufCapacity;
+    }
 
-    @Override
-    public void putByte(byte b) {
-        ensureCapacity(1);
-        Unsafe.getUnsafe().putByte(bufPtr + writePos, b);
-        writePos++;
+    /**
+     * Gets the payload length of the current frame being built.
+     */
+    public int getCurrentPayloadLength() {
+        return writePos - payloadStartOffset;
     }
 
+    /**
+     * Gets the current write position (number of bytes written).
+     */
     @Override
-    public void putInt(int value) {
-        ensureCapacity(4);
-        Unsafe.getUnsafe().putInt(bufPtr + writePos, value);
-        writePos += 4;
+    public int getPosition() {
+        return writePos;
     }
 
-    @Override
-    public void putLong(long value) {
-        ensureCapacity(8);
-        Unsafe.getUnsafe().putLong(bufPtr + writePos, value);
-        writePos += 8;
+    /**
+     * Gets the current write position (total bytes written since last reset).
+     */
+    public int getWritePos() {
+        return writePos;
     }
 
+    /**
+     * Patches an int value at the specified offset.
+     */
     @Override
-    public void putDouble(double value) {
-        ensureCapacity(8);
-        Unsafe.getUnsafe().putDouble(bufPtr + writePos, value);
-        writePos += 8;
+    public void patchInt(int offset, int value) {
+        Unsafe.getUnsafe().putInt(bufPtr + offset, value);
+    }
+
+    /**
+     * Writes an ASCII string.
+     */
+    public void putAscii(CharSequence cs) {
+        if (cs == null) {
+            return;
+        }
+        int len = cs.length();
+        ensureCapacity(len);
+        for (int i = 0; i < len; i++) {
+            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i));
+        }
+        writePos += len;
     }
 
     @Override
@@ -186,15 +270,32 @@ public void putBlockOfBytes(long from, long len) {
         writePos += (int) len;
     }
 
-    // === Additional write methods (not in ArrayBufferAppender but useful) ===
+    @Override
+    public void putByte(byte b) {
+        ensureCapacity(1);
+        Unsafe.getUnsafe().putByte(bufPtr + writePos, b);
+        writePos++;
+    }
 
     /**
-     * Writes a short value in little-endian format.
+     * Writes raw bytes from a byte array.
      */
-    public void putShort(short value) {
-        ensureCapacity(2);
-        Unsafe.getUnsafe().putShort(bufPtr + writePos, value);
-        writePos += 2;
+    public void putBytes(byte[] bytes, int offset, int length) {
+        if (length <= 0) {
+            return;
+        }
+        ensureCapacity(length);
+        for (int i = 0; i < length; i++) {
+            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]);
+        }
+        writePos += length;
+    }
+
+    @Override
+    public void putDouble(double value) {
+        ensureCapacity(8);
+        Unsafe.getUnsafe().putDouble(bufPtr + writePos, value);
+        writePos += 8;
     }
 
     /**
@@ -206,6 +307,20 @@ public void putFloat(float value) {
         writePos += 4;
     }
 
+    @Override
+    public void putInt(int value) {
+        ensureCapacity(4);
+        Unsafe.getUnsafe().putInt(bufPtr + writePos, value);
+        writePos += 4;
+    }
+
+    @Override
+    public void putLong(long value) {
+        ensureCapacity(8);
+        Unsafe.getUnsafe().putLong(bufPtr + writePos, value);
+        writePos += 8;
+    }
+
     /**
      * Writes a long value in big-endian format.
      */
@@ -216,46 +331,12 @@ public void putLongBE(long value) {
     }
 
     /**
-     * Writes raw bytes from a byte array.
-     */
-    public void putBytes(byte[] bytes, int offset, int length) {
-        if (length <= 0) {
-            return;
-        }
-        ensureCapacity(length);
-        for (int i = 0; i < length; i++) {
-            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]);
-        }
-        writePos += length;
-    }
-
-    /**
-     * Writes an ASCII string.
-     */
-    public void putAscii(CharSequence cs) {
-        if (cs == null) {
-            return;
-        }
-        int len = cs.length();
-        ensureCapacity(len);
-        for (int i = 0; i < len; i++) {
-            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i));
-        }
-        writePos += len;
-    }
-
-    // === QwpBufferWriter Implementation ===
-
-    /**
-     * Writes an unsigned variable-length integer (LEB128 encoding).
+     * Writes a short value in little-endian format.
      */
-    @Override
-    public void putVarint(long value) {
-        while (value > 0x7F) {
-            putByte((byte) ((value & 0x7F) | 0x80));
-            value >>>= 7;
-        }
-        putByte((byte) value);
+    public void putShort(short value) {
+        ensureCapacity(2);
+        Unsafe.getUnsafe().putShort(bufPtr + writePos, value);
+        writePos += 2;
     }
 
     /**
@@ -303,106 +384,79 @@ public void putUtf8(String value) {
     }
 
     /**
-     * Patches an int value at the specified offset.
+     * Writes an unsigned variable-length integer (LEB128 encoding).
      */
     @Override
-    public void patchInt(int offset, int value) {
-        Unsafe.getUnsafe().putInt(bufPtr + offset, value);
+    public void putVarint(long value) {
+        while (value > 0x7F) {
+            putByte((byte) ((value & 0x7F) | 0x80));
+            value >>>= 7;
+        }
+        putByte((byte) value);
     }
 
     /**
-     * Skips the specified number of bytes, advancing the position.
+     * Resets the buffer for reuse. Does not deallocate memory.
      */
-    @Override
-    public void skip(int bytes) {
-        ensureCapacity(bytes);
-        writePos += bytes;
+    public void reset() {
+        writePos = 0;
+        frameStartOffset = 0;
+        payloadStartOffset = 0;
     }
 
     /**
-     * Gets the current write position (number of bytes written).
+     * Skips the specified number of bytes, advancing the position.
      */
     @Override
-    public int getPosition() {
-        return writePos;
-    }
-
-    // === Frame Building ===
-
-    /**
-     * Begins a new binary WebSocket frame. Reserves space for the maximum header size.
-     * After calling this method, use ArrayBufferAppender methods to write the payload.
-     */
-    public void beginBinaryFrame() {
-        beginFrame(WebSocketOpcode.BINARY);
-    }
-
-    /**
-     * Begins a new text WebSocket frame. Reserves space for the maximum header size.
-     */
-    public void beginTextFrame() {
-        beginFrame(WebSocketOpcode.TEXT);
-    }
-
-    /**
-     * Begins a new WebSocket frame with the specified opcode.
-     *
-     * @param opcode the frame opcode
-     */
-    public void beginFrame(int opcode) {
-        frameStartOffset = writePos;
-        // Reserve maximum header space
-        ensureCapacity(MAX_HEADER_SIZE);
-        writePos += MAX_HEADER_SIZE;
-        payloadStartOffset = writePos;
+    public void skip(int bytes) {
+        ensureCapacity(bytes);
+        writePos += bytes;
     }
 
     /**
-     * Finishes the current binary frame, writing the header and applying masking.
-     * Returns information about where to find the complete frame in the buffer.
-     * 

-     * IMPORTANT: Only call this after all payload writes are complete. The buffer
-     * pointer is stable after this call (no more reallocations for this frame).
+     * Writes a complete close frame.
      *
-     * @return frame info containing offset and length for sending
-     */
-    public FrameInfo endBinaryFrame() {
-        return endFrame(WebSocketOpcode.BINARY);
-    }
-
-    /**
-     * Finishes the current text frame, writing the header and applying masking.
+     * @param code   close status code (e.g., 1000 for normal closure)
+     * @param reason optional reason string (may be null)
+     * @return frame info for sending
      */
-    public FrameInfo endTextFrame() {
-        return endFrame(WebSocketOpcode.TEXT);
-    }
+    public FrameInfo writeCloseFrame(int code, String reason) {
+        int payloadLen = 2; // status code
+        byte[] reasonBytes = null;
+        if (reason != null && !reason.isEmpty()) {
+            reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+            payloadLen += reasonBytes.length;
+        }
 
-    /**
-     * Finishes the current frame with the specified opcode.
-     *
-     * @param opcode the frame opcode
-     * @return frame info containing offset and length for sending
-     */
-    public FrameInfo endFrame(int opcode) {
-        int payloadLen = writePos - payloadStartOffset;
+        if (payloadLen > 125) {
+            throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']');
+        }
 
-        // Calculate actual header size (with mask key for client frames)
-        int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true);
-        int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize;
-        int actualFrameStart = frameStartOffset + unusedSpace;
+        int frameStart = writePos;
+        int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+        ensureCapacity(headerSize + payloadLen);
 
-        // Generate mask key
         int maskKey = rnd.nextInt();
+        int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey);
+        writePos += written;
 
-        // Write header at actual position (after unused space)
-        WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey);
+        // Write status code (big-endian)
+        long payloadStart = bufPtr + writePos;
+        Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF));
+        Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF));
+        writePos += 2;
 
-        // Apply mask to payload
-        if (payloadLen > 0) {
-            WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey);
+        // Write reason if present
+        if (reasonBytes != null) {
+            for (byte reasonByte : reasonBytes) {
+                Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte);
+            }
         }
 
-        return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen);
+        // Mask the payload (including status code and reason)
+        WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey);
+
+        return frameInfo.set(frameStart, headerSize + payloadLen);
     }
 
     /**
@@ -473,89 +527,20 @@ public FrameInfo writePongFrame(long payloadPtr, int payloadLen) {
         return frameInfo.set(frameStart, headerSize + payloadLen);
     }
 
-    /**
-     * Writes a complete close frame.
-     *
-     * @param code   close status code (e.g., 1000 for normal closure)
-     * @param reason optional reason string (may be null)
-     * @return frame info for sending
-     */
-    public FrameInfo writeCloseFrame(int code, String reason) {
-        int payloadLen = 2; // status code
-        byte[] reasonBytes = null;
-        if (reason != null && !reason.isEmpty()) {
-            reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8);
-            payloadLen += reasonBytes.length;
-        }
-
-        if (payloadLen > 125) {
-            throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']');
-        }
-
-        int frameStart = writePos;
-        int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
-        ensureCapacity(headerSize + payloadLen);
-
-        int maskKey = rnd.nextInt();
-        int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey);
-        writePos += written;
-
-        // Write status code (big-endian)
-        long payloadStart = bufPtr + writePos;
-        Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF));
-        Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF));
-        writePos += 2;
-
-        // Write reason if present
-        if (reasonBytes != null) {
-            for (byte reasonByte : reasonBytes) {
-                Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte);
-            }
+    private void grow(long requiredCapacity) {
+        if (requiredCapacity > maxBufferSize) {
+            throw new HttpClientException("WebSocket buffer size exceeded maximum [required=")
+                    .put(requiredCapacity)
+                    .put(", max=")
+                    .put(maxBufferSize)
+                    .put(']');
         }
-
-        // Mask the payload (including status code and reason)
-        WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey);
-
-        return frameInfo.set(frameStart, headerSize + payloadLen);
-    }
-
-    // === Buffer State ===
-
-    /**
-     * Gets the buffer pointer. Only use this for reading after frame is complete.
-     */
-    public long getBufferPtr() {
-        return bufPtr;
-    }
-
-    /**
-     * Gets the current buffer capacity.
-     */
-    public int getCapacity() {
-        return bufCapacity;
-    }
-
-    /**
-     * Gets the current write position (total bytes written since last reset).
-     */
-    public int getWritePos() {
-        return writePos;
-    }
-
-    /**
-     * Gets the payload length of the current frame being built.
-     */
-    public int getCurrentPayloadLength() {
-        return writePos - payloadStartOffset;
-    }
-
-    /**
-     * Resets the buffer for reuse. Does not deallocate memory.
-     */
-    public void reset() {
-        writePos = 0;
-        frameStartOffset = 0;
-        payloadStartOffset = 0;
+        int newCapacity = Math.min(
+                Numbers.ceilPow2((int) requiredCapacity),
+                maxBufferSize
+        );
+        bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
+        bufCapacity = newCapacity;
     }
 
     /**
@@ -564,15 +549,14 @@ public void reset() {
      * extract values before calling any end*Frame() method again.
      */
     public static final class FrameInfo {
-        /**
-         * Offset from buffer start where the frame begins.
-         */
-        public int offset;
-
         /**
          * Total length of the frame (header + payload).
          */
         public int length;
+        /**
+         * Offset from buffer start where the frame begins.
+         */
+        public int offset;
 
         FrameInfo set(int offset, int length) {
             this.offset = offset;
diff --git a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
index 8b4caf0..1f0cc05 100644
--- a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
@@ -65,6 +65,7 @@ public class JsonLexer implements Mutable, Closeable {
     private boolean quoted = false;
     private int state = S_START;
     private boolean useCache = false;
+
     public JsonLexer(int cacheSize, int cacheSizeLimit) {
         this.cacheSizeLimit = cacheSizeLimit;
         // if cacheSizeLimit is 0 or negative, the cache is disabled
@@ -398,4 +399,4 @@ private void utf8DecodeCacheAndBuffer(long lo, long hi, int position) throws Jso
         unquotedTerminators.add('{');
         unquotedTerminators.add('[');
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
index 0ed3937..11274a1 100644
--- a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
@@ -368,7 +368,7 @@ private static int findEOL(long ptr, int len) {
 
     private byte[] receiveChallengeBytes() {
         int n = 0;
-        for (;;) {
+        for (; ; ) {
             int rc = lineChannel.receive(ptr + n, capacity - n);
             if (rc < 0) {
                 int errno = lineChannel.errno();
@@ -505,4 +505,4 @@ protected AbstractLineSender writeFieldName(CharSequence name) {
         }
         throw new LineSenderException("table expected");
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
index 4d75211..ace342e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
@@ -38,8 +38,8 @@
  */
 public class GlobalSymbolDictionary {
 
-    private final CharSequenceIntHashMap symbolToId;
     private final ObjList idToSymbol;
+    private final CharSequenceIntHashMap symbolToId;
 
     public GlobalSymbolDictionary() {
         this(64); // Default initial capacity
@@ -50,6 +50,40 @@ public GlobalSymbolDictionary(int initialCapacity) {
         this.idToSymbol = new ObjList<>(initialCapacity);
     }
 
+    /**
+     * Clears all symbols from the dictionary.
+     * 

+     * After clearing, the next symbol added will get ID 0.
+     */
+    public void clear() {
+        symbolToId.clear();
+        idToSymbol.clear();
+    }
+
+    /**
+     * Checks if the dictionary contains the given symbol.
+     *
+     * @param symbol the symbol to check
+     * @return true if the symbol exists in the dictionary
+     */
+    public boolean contains(String symbol) {
+        return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE;
+    }
+
+    /**
+     * Gets the ID for an existing symbol, or -1 if not found.
+     *
+     * @param symbol the symbol string
+     * @return the symbol ID, or -1 if not in dictionary
+     */
+    public int getId(String symbol) {
+        if (symbol == null) {
+            return -1;
+        }
+        int id = symbolToId.get(symbol);
+        return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id;
+    }
+
     /**
      * Gets or adds a symbol to the dictionary.
      * 

@@ -91,48 +125,6 @@ public String getSymbol(int id) {
         return idToSymbol.getQuick(id);
     }
 
-    /**
-     * Gets the ID for an existing symbol, or -1 if not found.
-     *
-     * @param symbol the symbol string
-     * @return the symbol ID, or -1 if not in dictionary
-     */
-    public int getId(String symbol) {
-        if (symbol == null) {
-            return -1;
-        }
-        int id = symbolToId.get(symbol);
-        return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id;
-    }
-
-    /**
-     * Returns the number of symbols in the dictionary.
-     *
-     * @return dictionary size
-     */
-    public int size() {
-        return idToSymbol.size();
-    }
-
-    /**
-     * Checks if the dictionary is empty.
-     *
-     * @return true if no symbols have been added
-     */
-    public boolean isEmpty() {
-        return idToSymbol.size() == 0;
-    }
-
-    /**
-     * Checks if the dictionary contains the given symbol.
-     *
-     * @param symbol the symbol to check
-     * @return true if the symbol exists in the dictionary
-     */
-    public boolean contains(String symbol) {
-        return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE;
-    }
-
     /**
      * Gets the symbols in the given ID range [fromId, toId).
      * 

@@ -162,12 +154,20 @@ public String[] getSymbolsInRange(int fromId, int toId) {
     }
 
     /**
-     * Clears all symbols from the dictionary.
-     * 

-     * After clearing, the next symbol added will get ID 0.
+     * Checks if the dictionary is empty.
+     *
+     * @return true if no symbols have been added
      */
-    public void clear() {
-        symbolToId.clear();
-        idToSymbol.clear();
+    public boolean isEmpty() {
+        return idToSymbol.size() == 0;
+    }
+
+    /**
+     * Returns the number of symbols in the dictionary.
+     *
+     * @return dictionary size
+     */
+    public int size() {
+        return idToSymbol.size();
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
index cffd93e..869d242 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
@@ -57,38 +57,29 @@
  */
 public class InFlightWindow {
 
-    private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class);
-
-    public static final int DEFAULT_WINDOW_SIZE = 8;
     public static final long DEFAULT_TIMEOUT_MS = 30_000;
-
+    public static final int DEFAULT_WINDOW_SIZE = 8;
+    private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class);
+    private static final long PARK_NANOS = 100_000; // 100 microseconds
     // Spin parameters
     private static final int SPIN_TRIES = 100;
-    private static final long PARK_NANOS = 100_000; // 100 microseconds
-
+    // Error state
+    private final AtomicReference lastError = new AtomicReference<>();
     private final int maxWindowSize;
     private final long timeoutMs;
-
+    private volatile long failedBatchId = -1;
+    // highestAcked: the sequence number of the last acknowledged batch (cumulative)
+    private volatile long highestAcked = -1;
     // Core state
     // highestSent: the sequence number of the last batch added to the window
     private volatile long highestSent = -1;
-
-    // highestAcked: the sequence number of the last acknowledged batch (cumulative)
-    private volatile long highestAcked = -1;
-
-    // Error state
-    private final AtomicReference lastError = new AtomicReference<>();
-    private volatile long failedBatchId = -1;
-
-    // Thread waiting for space (sender thread)
-    private volatile Thread waitingForSpace;
-
-    // Thread waiting for empty (flush thread)
-    private volatile Thread waitingForEmpty;
-
     // Statistics (not strictly accurate under contention, but good enough for monitoring)
     private volatile long totalAcked = 0;
     private volatile long totalFailed = 0;
+    // Thread waiting for empty (flush thread)
+    private volatile Thread waitingForEmpty;
+    // Thread waiting for space (sender thread)
+    private volatile Thread waitingForSpace;
 
     /**
      * Creates a new InFlightWindow with default configuration.
@@ -112,37 +103,64 @@ public InFlightWindow(int maxWindowSize, long timeoutMs) {
     }
 
     /**
-     * Checks if there's space in the window for another batch.
-     * Wait-free operation.
+     * Acknowledges a batch, removing it from the in-flight window.
+     * 

+     * For sequential batch IDs, this is a cumulative acknowledgment -
+     * acknowledging batch N means all batches up to N are acknowledged.
+     * 

+     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
      *
-     * @return true if there's space, false if window is full
+     * @param batchId the batch ID that was acknowledged
+     * @return true if the batch was in flight, false if already acknowledged
      */
-    public boolean hasWindowSpace() {
-        return getInFlightCount() < maxWindowSize;
+    public boolean acknowledge(long batchId) {
+        return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId;
     }
 
     /**
-     * Tries to add a batch to the in-flight window without blocking.
-     * Lock-free, assuming single producer for highestSent.
+     * Acknowledges all batches up to and including the given sequence (cumulative ACK).
+     * Lock-free with single consumer.
+     * 

+     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
      *
-     * Called by: async producer (WebSocket I/O thread) before sending a batch.
-     * @param batchId the batch ID to track (must be sequential)
-     * @return true if added, false if window is full
+     * @param sequence the highest acknowledged sequence
+     * @return the number of batches acknowledged
      */
-    public boolean tryAddInFlight(long batchId) {
-        // Check window space first
+    public int acknowledgeUpTo(long sequence) {
         long sent = highestSent;
-        long acked = highestAcked;
 
-        if (sent - acked >= maxWindowSize) {
-            return false;
+        // Nothing to acknowledge if window is empty or sequence is beyond what's sent
+        if (sent < 0) {
+            return 0; // No batches have been sent
         }
 
-        // Sequential caller: just publish the new highestSent
-        highestSent = batchId;
+        // Cap sequence at highestSent - can't acknowledge what hasn't been sent
+        long effectiveSequence = Math.min(sequence, sent);
 
-        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
-        return true;
+        long prevAcked = highestAcked;
+        if (effectiveSequence <= prevAcked) {
+            // Already acknowledged up to this point
+            return 0;
+        }
+        highestAcked = effectiveSequence;
+
+        int acknowledged = (int) (effectiveSequence - prevAcked);
+        totalAcked += acknowledged;
+
+        LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount());
+
+        // Wake up waiting threads
+        Thread waiter = waitingForSpace;
+        if (waiter != null) {
+            LockSupport.unpark(waiter);
+        }
+
+        waiter = waitingForEmpty;
+        if (waiter != null && getInFlightCount() == 0) {
+            LockSupport.unpark(waiter);
+        }
+
+        return acknowledged;
     }
 
     /**
@@ -156,8 +174,9 @@ public boolean tryAddInFlight(long batchId) {
      * it must ensure ACKs are processed on another thread; a single-threaded caller
      * with window>1 would deadlock by parking while also being the only thread that
      * can advance {@link #acknowledgeUpTo(long)}.
-     *
+     * 

      * Called by: sync sender thread before sending a batch (window=1).
+     *
      * @param batchId the batch ID to track
      * @throws LineSenderException if timeout occurs or an error was reported
      */
@@ -210,85 +229,69 @@ public void addInFlight(long batchId) {
         }
     }
 
-    private boolean tryAddInFlightInternal(long batchId) {
-        long sent = highestSent;
-        long acked = highestAcked;
-
-        if (sent - acked >= maxWindowSize) {
-            return false;
-        }
-
-        // For sequential IDs, we just update highestSent
-        // The caller guarantees batchId is the next in sequence
-        highestSent = batchId;
-
-        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
-        return true;
-    }
-
     /**
-     * Acknowledges a batch, removing it from the in-flight window.
+     * Waits until all in-flight batches are acknowledged.
      * 

-     * For sequential batch IDs, this is a cumulative acknowledgment -
-     * acknowledging batch N means all batches up to N are acknowledged.
-     *
-     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
-     * @param batchId the batch ID that was acknowledged
-     * @return true if the batch was in flight, false if already acknowledged
-     */
-    public boolean acknowledge(long batchId) {
-        return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId;
-    }
-
-    /**
-     * Acknowledges all batches up to and including the given sequence (cumulative ACK).
-     * Lock-free with single consumer.
+     * Called by flush() to ensure all data is confirmed.
+     * 

+     * Called by: waiter (flush thread), while producer/acker thread progresses.
      *
-     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
-     * @param sequence the highest acknowledged sequence
-     * @return the number of batches acknowledged
+     * @throws LineSenderException if timeout occurs or an error was reported
      */
-    public int acknowledgeUpTo(long sequence) {
-        long sent = highestSent;
+    public void awaitEmpty() {
+        checkError();
 
-        // Nothing to acknowledge if window is empty or sequence is beyond what's sent
-        if (sent < 0) {
-            return 0; // No batches have been sent
+        // Fast path: already empty
+        if (getInFlightCount() == 0) {
+            LOG.debug("Window already empty");
+            return;
         }
 
-        // Cap sequence at highestSent - can't acknowledge what hasn't been sent
-        long effectiveSequence = Math.min(sequence, sent);
-
-        long prevAcked = highestAcked;
-        if (effectiveSequence <= prevAcked) {
-            // Already acknowledged up to this point
-            return 0;
-        }
-        highestAcked = effectiveSequence;
+        long deadline = System.currentTimeMillis() + timeoutMs;
+        int spins = 0;
 
-        int acknowledged = (int) (effectiveSequence - prevAcked);
-        totalAcked += acknowledged;
+        // Register as waiting thread
+        waitingForEmpty = Thread.currentThread();
+        try {
+            while (getInFlightCount() > 0) {
+                checkError();
 
-        LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount());
+                long remaining = deadline - System.currentTimeMillis();
+                if (remaining <= 0) {
+                    throw new LineSenderException("Timeout waiting for batch acknowledgments, " +
+                            getInFlightCount() + " batches still in flight");
+                }
 
-        // Wake up waiting threads
-        Thread waiter = waitingForSpace;
-        if (waiter != null) {
-            LockSupport.unpark(waiter);
-        }
+                if (spins < SPIN_TRIES) {
+                    Thread.onSpinWait();
+                    spins++;
+                } else {
+                    LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000));
+                    if (Thread.interrupted()) {
+                        throw new LineSenderException("Interrupted while waiting for acknowledgments");
+                    }
+                }
+            }
 
-        waiter = waitingForEmpty;
-        if (waiter != null && getInFlightCount() == 0) {
-            LockSupport.unpark(waiter);
+            LOG.debug("Window empty, all batches ACKed");
+        } finally {
+            waitingForEmpty = null;
         }
+    }
 
-        return acknowledged;
+    /**
+     * Clears the error state.
+     */
+    public void clearError() {
+        lastError.set(null);
+        failedBatchId = -1;
     }
 
     /**
      * Marks a batch as failed, setting an error that will be propagated to waiters.
-     *
+     * 

      * Called by: acker (WebSocket I/O thread) on error response or send failure.
+     *
      * @param batchId the batch ID that failed
      * @param error   the error that occurred
      */
@@ -324,55 +327,6 @@ public void failAll(Throwable error) {
         wakeWaiters();
     }
 
-    /**
-     * Waits until all in-flight batches are acknowledged.
-     * 

-     * Called by flush() to ensure all data is confirmed.
-     *
-     * Called by: waiter (flush thread), while producer/acker thread progresses.
-     * @throws LineSenderException if timeout occurs or an error was reported
-     */
-    public void awaitEmpty() {
-        checkError();
-
-        // Fast path: already empty
-        if (getInFlightCount() == 0) {
-            LOG.debug("Window already empty");
-            return;
-        }
-
-        long deadline = System.currentTimeMillis() + timeoutMs;
-        int spins = 0;
-
-        // Register as waiting thread
-        waitingForEmpty = Thread.currentThread();
-        try {
-            while (getInFlightCount() > 0) {
-                checkError();
-
-                long remaining = deadline - System.currentTimeMillis();
-                if (remaining <= 0) {
-                    throw new LineSenderException("Timeout waiting for batch acknowledgments, " +
-                            getInFlightCount() + " batches still in flight");
-                }
-
-                if (spins < SPIN_TRIES) {
-                    Thread.onSpinWait();
-                    spins++;
-                } else {
-                    LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000));
-                    if (Thread.interrupted()) {
-                        throw new LineSenderException("Interrupted while waiting for acknowledgments");
-                    }
-                }
-            }
-
-            LOG.debug("Window empty, all batches ACKed");
-        } finally {
-            waitingForEmpty = null;
-        }
-    }
-
     /**
      * Returns the current number of batches in flight.
      * Wait-free operation.
@@ -385,19 +339,10 @@ public int getInFlightCount() {
     }
 
     /**
-     * Returns true if the window is empty.
-     * Wait-free operation.
-     */
-    public boolean isEmpty() {
-        return getInFlightCount() == 0;
-    }
-
-    /**
-     * Returns true if the window is full.
-     * Wait-free operation.
+     * Returns the last error, or null if no error.
      */
-    public boolean isFull() {
-        return getInFlightCount() >= maxWindowSize;
+    public Throwable getLastError() {
+        return lastError.get();
     }
 
     /**
@@ -422,18 +367,29 @@ public long getTotalFailed() {
     }
 
     /**
-     * Returns the last error, or null if no error.
+     * Checks if there's space in the window for another batch.
+     * Wait-free operation.
+     *
+     * @return true if there's space, false if window is full
      */
-    public Throwable getLastError() {
-        return lastError.get();
+    public boolean hasWindowSpace() {
+        return getInFlightCount() < maxWindowSize;
     }
 
     /**
-     * Clears the error state.
+     * Returns true if the window is empty.
+     * Wait-free operation.
      */
-    public void clearError() {
-        lastError.set(null);
-        failedBatchId = -1;
+    public boolean isEmpty() {
+        return getInFlightCount() == 0;
+    }
+
+    /**
+     * Returns true if the window is full.
+     * Wait-free operation.
+     */
+    public boolean isFull() {
+        return getInFlightCount() >= maxWindowSize;
     }
 
     /**
@@ -448,6 +404,31 @@ public void reset() {
         wakeWaiters();
     }
 
+    /**
+     * Tries to add a batch to the in-flight window without blocking.
+     * Lock-free, assuming single producer for highestSent.
+     * 

+     * Called by: async producer (WebSocket I/O thread) before sending a batch.
+     *
+     * @param batchId the batch ID to track (must be sequential)
+     * @return true if added, false if window is full
+     */
+    public boolean tryAddInFlight(long batchId) {
+        // Check window space first
+        long sent = highestSent;
+        long acked = highestAcked;
+
+        if (sent - acked >= maxWindowSize) {
+            return false;
+        }
+
+        // Sequential caller: just publish the new highestSent
+        highestSent = batchId;
+
+        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
+        return true;
+    }
+
     private void checkError() {
         Throwable error = lastError.get();
         if (error != null) {
@@ -455,6 +436,22 @@ private void checkError() {
         }
     }
 
+    private boolean tryAddInFlightInternal(long batchId) {
+        long sent = highestSent;
+        long acked = highestAcked;
+
+        if (sent - acked >= maxWindowSize) {
+            return false;
+        }
+
+        // For sequential IDs, we just update highestSent
+        // The caller guarantees batchId is the next in sequence
+        highestSent = batchId;
+
+        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
+        return true;
+    }
+
     private void wakeWaiters() {
         Thread waiter = waitingForSpace;
         if (waiter != null) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
index 4ef2af4..e7efe6f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
@@ -57,37 +57,30 @@ public class MicrobatchBuffer implements QuietCloseable {
 
     // Buffer states
     public static final int STATE_FILLING = 0;
+    public static final int STATE_RECYCLED = 3;
     public static final int STATE_SEALED = 1;
     public static final int STATE_SENDING = 2;
-    public static final int STATE_RECYCLED = 3;
-
+    private static final AtomicLong nextBatchId = new AtomicLong();
+    private final long maxAgeNanos;
+    private final int maxBytes;
     // Flush trigger thresholds
     private final int maxRows;
-    private final int maxBytes;
-    private final long maxAgeNanos;
-
-    // Native memory buffer
-    private long bufferPtr;
+    // Batch identification
+    private long batchId;
     private int bufferCapacity;
     private int bufferPos;
-
-    // Row tracking
-    private int rowCount;
+    // Native memory buffer
+    private long bufferPtr;
     private long firstRowTimeNanos;
-
     // Symbol tracking for delta encoding
     private int maxSymbolId = -1;
-
-    // Batch identification
-    private long batchId;
-    private static final AtomicLong nextBatchId = new AtomicLong();
-
-    // State machine
-    private volatile int state = STATE_FILLING;
-
     // For waiting on recycle (user thread waits for I/O thread to finish)
     // CountDownLatch is not resettable, so we create a new instance on reset()
     private volatile CountDownLatch recycleLatch = new CountDownLatch(1);
+    // Row tracking
+    private int rowCount;
+    // State machine
+    private volatile int state = STATE_FILLING;
 
     /**
      * Creates a new MicrobatchBuffer with specified flush thresholds.
@@ -121,44 +114,59 @@ public MicrobatchBuffer(int initialCapacity) {
         this(initialCapacity, 0, 0, 0);
     }
 
-    // ==================== DATA OPERATIONS ====================
-
     /**
-     * Returns the buffer pointer for writing data.
-     * Only valid when state is FILLING.
-     */
-    public long getBufferPtr() {
-        return bufferPtr;
-    }
-
-    /**
-     * Returns the current write position in the buffer.
+     * Returns a human-readable name for the given state.
      */
-    public int getBufferPos() {
-        return bufferPos;
+    public static String stateName(int state) {
+        switch (state) {
+            case STATE_FILLING:
+                return "FILLING";
+            case STATE_SEALED:
+                return "SEALED";
+            case STATE_SENDING:
+                return "SENDING";
+            case STATE_RECYCLED:
+                return "RECYCLED";
+            default:
+                return "UNKNOWN(" + state + ")";
+        }
     }
 
     /**
-     * Returns the buffer capacity.
+     * Waits for the buffer to be recycled (transition to RECYCLED state).
+     * Only the user thread should call this.
      */
-    public int getBufferCapacity() {
-        return bufferCapacity;
+    public void awaitRecycled() {
+        try {
+            recycleLatch.await();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
     }
 
     /**
-     * Sets the buffer position after external writes.
-     * Only valid when state is FILLING.
+     * Waits for the buffer to be recycled with a timeout.
      *
-     * @param pos new position
+     * @param timeout the maximum time to wait
+     * @param unit    the time unit
+     * @return true if recycled, false if timeout elapsed
      */
-    public void setBufferPos(int pos) {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot set position when state is " + stateName(state));
+    public boolean awaitRecycled(long timeout, TimeUnit unit) {
+        try {
+            return recycleLatch.await(timeout, unit);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return false;
         }
-        if (pos < 0 || pos > bufferCapacity) {
-            throw new IllegalArgumentException("Position out of bounds: " + pos);
+    }
+
+    @Override
+    public void close() {
+        if (bufferPtr != 0) {
+            Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS);
+            bufferPtr = 0;
+            bufferCapacity = 0;
         }
-        this.bufferPos = pos;
     }
 
     /**
@@ -180,67 +188,42 @@ public void ensureCapacity(int requiredCapacity) {
     }
 
     /**
-     * Writes bytes to the buffer at the current position.
-     * Grows the buffer if necessary.
-     *
-     * @param src    source address
-     * @param length number of bytes to write
-     */
-    public void write(long src, int length) {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot write when state is " + stateName(state));
-        }
-        ensureCapacity(bufferPos + length);
-        Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length);
-        bufferPos += length;
-    }
-
-    /**
-     * Writes a single byte to the buffer.
-     *
-     * @param b byte to write
+     * Returns the age of the first row in nanoseconds, or 0 if no rows.
      */
-    public void writeByte(byte b) {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot write when state is " + stateName(state));
+    public long getAgeNanos() {
+        if (rowCount == 0) {
+            return 0;
         }
-        ensureCapacity(bufferPos + 1);
-        Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b);
-        bufferPos++;
+        return System.nanoTime() - firstRowTimeNanos;
     }
 
     /**
-     * Increments the row count and records the first row time if this is the first row.
+     * Returns the batch ID for this buffer.
      */
-    public void incrementRowCount() {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot increment row count when state is " + stateName(state));
-        }
-        if (rowCount == 0) {
-            firstRowTimeNanos = System.nanoTime();
-        }
-        rowCount++;
+    public long getBatchId() {
+        return batchId;
     }
 
     /**
-     * Returns the number of rows in this buffer.
+     * Returns the buffer capacity.
      */
-    public int getRowCount() {
-        return rowCount;
+    public int getBufferCapacity() {
+        return bufferCapacity;
     }
 
     /**
-     * Returns true if the buffer has any data.
+     * Returns the current write position in the buffer.
      */
-    public boolean hasData() {
-        return bufferPos > 0;
+    public int getBufferPos() {
+        return bufferPos;
     }
 
     /**
-     * Returns the batch ID for this buffer.
+     * Returns the buffer pointer for writing data.
+     * Only valid when state is FILLING.
      */
-    public long getBatchId() {
-        return batchId;
+    public long getBufferPtr() {
+        return bufferPtr;
     }
 
     /**
@@ -252,39 +235,37 @@ public int getMaxSymbolId() {
     }
 
     /**
-     * Sets the maximum symbol ID used in this batch.
-     * Used for delta symbol dictionary tracking.
+     * Returns the number of rows in this buffer.
      */
-    public void setMaxSymbolId(int maxSymbolId) {
-        this.maxSymbolId = maxSymbolId;
+    public int getRowCount() {
+        return rowCount;
     }
 
-    // ==================== FLUSH TRIGGER CHECKS ====================
-
     /**
-     * Checks if the buffer should be flushed based on configured thresholds.
-     *
-     * @return true if any flush threshold is exceeded
+     * Returns the current state.
      */
-    public boolean shouldFlush() {
-        if (!hasData()) {
-            return false;
-        }
-        return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded();
+    public int getState() {
+        return state;
     }
 
     /**
-     * Checks if the row count limit has been exceeded.
+     * Returns true if the buffer has any data.
      */
-    public boolean isRowLimitExceeded() {
-        return maxRows > 0 && rowCount >= maxRows;
+    public boolean hasData() {
+        return bufferPos > 0;
     }
 
     /**
-     * Checks if the byte size limit has been exceeded.
+     * Increments the row count and records the first row time if this is the first row.
      */
-    public boolean isByteLimitExceeded() {
-        return maxBytes > 0 && bufferPos >= maxBytes;
+    public void incrementRowCount() {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot increment row count when state is " + stateName(state));
+        }
+        if (rowCount == 0) {
+            firstRowTimeNanos = System.nanoTime();
+        }
+        rowCount++;
     }
 
     /**
@@ -299,22 +280,10 @@ public boolean isAgeLimitExceeded() {
     }
 
     /**
-     * Returns the age of the first row in nanoseconds, or 0 if no rows.
-     */
-    public long getAgeNanos() {
-        if (rowCount == 0) {
-            return 0;
-        }
-        return System.nanoTime() - firstRowTimeNanos;
-    }
-
-    // ==================== STATE MACHINE ====================
-
-    /**
-     * Returns the current state.
+     * Checks if the byte size limit has been exceeded.
      */
-    public int getState() {
-        return state;
+    public boolean isByteLimitExceeded() {
+        return maxBytes > 0 && bufferPos >= maxBytes;
     }
 
     /**
@@ -325,17 +294,11 @@ public boolean isFilling() {
     }
 
     /**
-     * Returns true if the buffer is in SEALED state (ready to send).
-     */
-    public boolean isSealed() {
-        return state == STATE_SEALED;
-    }
-
-    /**
-     * Returns true if the buffer is in SENDING state (being sent by I/O thread).
+     * Returns true if the buffer is currently in use (not available for the user thread).
      */
-    public boolean isSending() {
-        return state == STATE_SENDING;
+    public boolean isInUse() {
+        int s = state;
+        return s == STATE_SEALED || s == STATE_SENDING;
     }
 
     /**
@@ -346,53 +309,24 @@ public boolean isRecycled() {
     }
 
     /**
-     * Returns true if the buffer is currently in use (not available for the user thread).
-     */
-    public boolean isInUse() {
-        int s = state;
-        return s == STATE_SEALED || s == STATE_SENDING;
-    }
-
-    /**
-     * Seals the buffer, transitioning from FILLING to SEALED.
-     * After sealing, no more data can be written.
-     * Only the user thread should call this.
-     *
-     * @throws IllegalStateException if not in FILLING state
+     * Checks if the row count limit has been exceeded.
      */
-    public void seal() {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot seal buffer in state " + stateName(state));
-        }
-        state = STATE_SEALED;
+    public boolean isRowLimitExceeded() {
+        return maxRows > 0 && rowCount >= maxRows;
     }
 
     /**
-     * Rolls back a seal operation, transitioning from SEALED back to FILLING.
-     * 

-     * Used when enqueue fails after a buffer has been sealed but before ownership
-     * was transferred to the I/O thread.
-     *
-     * @throws IllegalStateException if not in SEALED state
+     * Returns true if the buffer is in SEALED state (ready to send).
      */
-    public void rollbackSealForRetry() {
-        if (state != STATE_SEALED) {
-            throw new IllegalStateException("Cannot rollback seal in state " + stateName(state));
-        }
-        state = STATE_FILLING;
+    public boolean isSealed() {
+        return state == STATE_SEALED;
     }
 
     /**
-     * Marks the buffer as being sent, transitioning from SEALED to SENDING.
-     * Only the I/O thread should call this.
-     *
-     * @throws IllegalStateException if not in SEALED state
+     * Returns true if the buffer is in SENDING state (being sent by I/O thread).
      */
-    public void markSending() {
-        if (state != STATE_SEALED) {
-            throw new IllegalStateException("Cannot mark sending in state " + stateName(state));
-        }
-        state = STATE_SENDING;
+    public boolean isSending() {
+        return state == STATE_SENDING;
     }
 
     /**
@@ -411,31 +345,16 @@ public void markRecycled() {
     }
 
     /**
-     * Waits for the buffer to be recycled (transition to RECYCLED state).
-     * Only the user thread should call this.
-     */
-    public void awaitRecycled() {
-        try {
-            recycleLatch.await();
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-        }
-    }
-
-    /**
-     * Waits for the buffer to be recycled with a timeout.
+     * Marks the buffer as being sent, transitioning from SEALED to SENDING.
+     * Only the I/O thread should call this.
      *
-     * @param timeout the maximum time to wait
-     * @param unit    the time unit
-     * @return true if recycled, false if timeout elapsed
+     * @throws IllegalStateException if not in SEALED state
      */
-    public boolean awaitRecycled(long timeout, TimeUnit unit) {
-        try {
-            return recycleLatch.await(timeout, unit);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            return false;
+    public void markSending() {
+        if (state != STATE_SEALED) {
+            throw new IllegalStateException("Cannot mark sending in state " + stateName(state));
         }
+        state = STATE_SENDING;
     }
 
     /**
@@ -458,35 +377,69 @@ public void reset() {
         recycleLatch = new CountDownLatch(1);
     }
 
-    // ==================== LIFECYCLE ====================
+    /**
+     * Rolls back a seal operation, transitioning from SEALED back to FILLING.
+     * 

+     * Used when enqueue fails after a buffer has been sealed but before ownership
+     * was transferred to the I/O thread.
+     *
+     * @throws IllegalStateException if not in SEALED state
+     */
+    public void rollbackSealForRetry() {
+        if (state != STATE_SEALED) {
+            throw new IllegalStateException("Cannot rollback seal in state " + stateName(state));
+        }
+        state = STATE_FILLING;
+    }
+
+    /**
+     * Seals the buffer, transitioning from FILLING to SEALED.
+     * After sealing, no more data can be written.
+     * Only the user thread should call this.
+     *
+     * @throws IllegalStateException if not in FILLING state
+     */
+    public void seal() {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot seal buffer in state " + stateName(state));
+        }
+        state = STATE_SEALED;
+    }
 
-    @Override
-    public void close() {
-        if (bufferPtr != 0) {
-            Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS);
-            bufferPtr = 0;
-            bufferCapacity = 0;
+    /**
+     * Sets the buffer position after external writes.
+     * Only valid when state is FILLING.
+     *
+     * @param pos new position
+     */
+    public void setBufferPos(int pos) {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot set position when state is " + stateName(state));
+        }
+        if (pos < 0 || pos > bufferCapacity) {
+            throw new IllegalArgumentException("Position out of bounds: " + pos);
         }
+        this.bufferPos = pos;
     }
 
-    // ==================== UTILITIES ====================
+    /**
+     * Sets the maximum symbol ID used in this batch.
+     * Used for delta symbol dictionary tracking.
+     */
+    public void setMaxSymbolId(int maxSymbolId) {
+        this.maxSymbolId = maxSymbolId;
+    }
 
     /**
-     * Returns a human-readable name for the given state.
+     * Checks if the buffer should be flushed based on configured thresholds.
+     *
+     * @return true if any flush threshold is exceeded
      */
-    public static String stateName(int state) {
-        switch (state) {
-            case STATE_FILLING:
-                return "FILLING";
-            case STATE_SEALED:
-                return "SEALED";
-            case STATE_SENDING:
-                return "SENDING";
-            case STATE_RECYCLED:
-                return "RECYCLED";
-            default:
-                return "UNKNOWN(" + state + ")";
+    public boolean shouldFlush() {
+        if (!hasData()) {
+            return false;
         }
+        return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded();
     }
 
     @Override
@@ -499,4 +452,34 @@ public String toString() {
                 ", capacity=" + bufferCapacity +
                 '}';
     }
+
+    /**
+     * Writes bytes to the buffer at the current position.
+     * Grows the buffer if necessary.
+     *
+     * @param src    source address
+     * @param length number of bytes to write
+     */
+    public void write(long src, int length) {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot write when state is " + stateName(state));
+        }
+        ensureCapacity(bufferPos + length);
+        Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length);
+        bufferPos += length;
+    }
+
+    /**
+     * Writes a single byte to the buffer.
+     *
+     * @param b byte to write
+     */
+    public void writeByte(byte b) {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot write when state is " + stateName(state));
+        }
+        ensureCapacity(bufferPos + 1);
+        Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b);
+        bufferPos++;
+    }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
index ee264ed..a11f70f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
@@ -54,6 +54,50 @@ public NativeBufferWriter(int initialCapacity) {
         this.position = 0;
     }
 
+    /**
+     * Returns the UTF-8 encoded length of a string.
+     */
+    public static int utf8Length(String s) {
+        if (s == null) return 0;
+        int len = 0;
+        for (int i = 0, n = s.length(); i < n; i++) {
+            char c = s.charAt(i);
+            if (c < 0x80) {
+                len++;
+            } else if (c < 0x800) {
+                len += 2;
+            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
+                i++;
+                len += 4;
+            } else {
+                len += 3;
+            }
+        }
+        return len;
+    }
+
+    @Override
+    public void close() {
+        if (bufferPtr != 0) {
+            Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT);
+            bufferPtr = 0;
+        }
+    }
+
+    /**
+     * Ensures the buffer has at least the specified additional capacity.
+     *
+     * @param needed additional bytes needed beyond current position
+     */
+    @Override
+    public void ensureCapacity(int needed) {
+        if (position + needed > capacity) {
+            int newCapacity = Math.max(capacity * 2, position + needed);
+            bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
+            capacity = newCapacity;
+        }
+    }
+
     /**
      * Returns the buffer pointer.
      */
@@ -62,6 +106,14 @@ public long getBufferPtr() {
         return bufferPtr;
     }
 
+    /**
+     * Returns the current buffer capacity.
+     */
+    @Override
+    public int getCapacity() {
+        return capacity;
+    }
+
     /**
      * Returns the current write position (number of bytes written).
      */
@@ -71,11 +123,22 @@ public int getPosition() {
     }
 
     /**
-     * Resets the buffer for reuse.
+     * Patches an int value at the specified offset.
+     * Used for updating length fields after writing content.
      */
     @Override
-    public void reset() {
-        position = 0;
+    public void patchInt(int offset, int value) {
+        Unsafe.getUnsafe().putInt(bufferPtr + offset, value);
+    }
+
+    /**
+     * Writes a block of bytes from native memory.
+     */
+    @Override
+    public void putBlockOfBytes(long from, long len) {
+        ensureCapacity((int) len);
+        Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len);
+        position += (int) len;
     }
 
     /**
@@ -89,13 +152,23 @@ public void putByte(byte value) {
     }
 
     /**
-     * Writes a short (2 bytes, little-endian).
+     * Writes a double (8 bytes, little-endian).
      */
     @Override
-    public void putShort(short value) {
-        ensureCapacity(2);
-        Unsafe.getUnsafe().putShort(bufferPtr + position, value);
-        position += 2;
+    public void putDouble(double value) {
+        ensureCapacity(8);
+        Unsafe.getUnsafe().putDouble(bufferPtr + position, value);
+        position += 8;
+    }
+
+    /**
+     * Writes a float (4 bytes, little-endian).
+     */
+    @Override
+    public void putFloat(float value) {
+        ensureCapacity(4);
+        Unsafe.getUnsafe().putFloat(bufferPtr + position, value);
+        position += 4;
     }
 
     /**
@@ -129,45 +202,13 @@ public void putLongBE(long value) {
     }
 
     /**
-     * Writes a float (4 bytes, little-endian).
-     */
-    @Override
-    public void putFloat(float value) {
-        ensureCapacity(4);
-        Unsafe.getUnsafe().putFloat(bufferPtr + position, value);
-        position += 4;
-    }
-
-    /**
-     * Writes a double (8 bytes, little-endian).
-     */
-    @Override
-    public void putDouble(double value) {
-        ensureCapacity(8);
-        Unsafe.getUnsafe().putDouble(bufferPtr + position, value);
-        position += 8;
-    }
-
-    /**
-     * Writes a block of bytes from native memory.
-     */
-    @Override
-    public void putBlockOfBytes(long from, long len) {
-        ensureCapacity((int) len);
-        Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len);
-        position += (int) len;
-    }
-
-    /**
-     * Writes a varint (unsigned LEB128).
+     * Writes a short (2 bytes, little-endian).
      */
     @Override
-    public void putVarint(long value) {
-        while (value > 0x7F) {
-            putByte((byte) ((value & 0x7F) | 0x80));
-            value >>>= 7;
-        }
-        putByte((byte) value);
+    public void putShort(short value) {
+        ensureCapacity(2);
+        Unsafe.getUnsafe().putShort(bufferPtr + position, value);
+        position += 2;
     }
 
     /**
@@ -216,42 +257,23 @@ public void putUtf8(String value) {
     }
 
     /**
-     * Returns the UTF-8 encoded length of a string.
-     */
-    public static int utf8Length(String s) {
-        if (s == null) return 0;
-        int len = 0;
-        for (int i = 0, n = s.length(); i < n; i++) {
-            char c = s.charAt(i);
-            if (c < 0x80) {
-                len++;
-            } else if (c < 0x800) {
-                len += 2;
-            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
-                i++;
-                len += 4;
-            } else {
-                len += 3;
-            }
-        }
-        return len;
-    }
-
-    /**
-     * Patches an int value at the specified offset.
-     * Used for updating length fields after writing content.
+     * Writes a varint (unsigned LEB128).
      */
     @Override
-    public void patchInt(int offset, int value) {
-        Unsafe.getUnsafe().putInt(bufferPtr + offset, value);
+    public void putVarint(long value) {
+        while (value > 0x7F) {
+            putByte((byte) ((value & 0x7F) | 0x80));
+            value >>>= 7;
+        }
+        putByte((byte) value);
     }
 
     /**
-     * Returns the current buffer capacity.
+     * Resets the buffer for reuse.
      */
     @Override
-    public int getCapacity() {
-        return capacity;
+    public void reset() {
+        position = 0;
     }
 
     /**
@@ -264,26 +286,4 @@ public int getCapacity() {
     public void skip(int bytes) {
         position += bytes;
     }
-
-    /**
-     * Ensures the buffer has at least the specified additional capacity.
-     *
-     * @param needed additional bytes needed beyond current position
-     */
-    @Override
-    public void ensureCapacity(int needed) {
-        if (position + needed > capacity) {
-            int newCapacity = Math.max(capacity * 2, position + needed);
-            bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
-            capacity = newCapacity;
-        }
-    }
-
-    @Override
-    public void close() {
-        if (bufferPtr != 0) {
-            Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT);
-            bufferPtr = 0;
-        }
-    }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
index 05ef0ea..c50cdf6 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
@@ -44,54 +44,56 @@
  */
 public interface QwpBufferWriter extends ArrayBufferAppender {
 
-    // === Primitive writes (little-endian) ===
-
     /**
-     * Writes a short (2 bytes, little-endian).
-     */
-    void putShort(short value);
-
-    /**
-     * Writes a float (4 bytes, little-endian).
+     * Returns the UTF-8 encoded length of a string.
+     *
+     * @param s the string (may be null)
+     * @return the number of bytes needed to encode the string as UTF-8
      */
-    void putFloat(float value);
-
-    // === Big-endian writes ===
+    static int utf8Length(String s) {
+        if (s == null) return 0;
+        int len = 0;
+        for (int i = 0, n = s.length(); i < n; i++) {
+            char c = s.charAt(i);
+            if (c < 0x80) {
+                len++;
+            } else if (c < 0x800) {
+                len += 2;
+            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
+                i++;
+                len += 4;
+            } else {
+                len += 3;
+            }
+        }
+        return len;
+    }
 
     /**
-     * Writes a long in big-endian byte order.
+     * Ensures the buffer has capacity for at least the specified
+     * additional bytes beyond the current position.
+     *
+     * @param additionalBytes number of additional bytes needed
      */
-    void putLongBE(long value);
-
-    // === Variable-length encoding ===
+    void ensureCapacity(int additionalBytes);
 
     /**
-     * Writes an unsigned variable-length integer (LEB128 encoding).
+     * Returns the native memory pointer to the buffer start.
      * 

-     * Each byte contains 7 bits of data with the high bit indicating
-     * whether more bytes follow.
+     * The returned pointer is valid until the next buffer growth operation.
+     * Use with care and only for reading completed data.
      */
-    void putVarint(long value);
-
-    // === String encoding ===
+    long getBufferPtr();
 
     /**
-     * Writes a length-prefixed UTF-8 string.
-     * 

-     * Format: varint length + UTF-8 bytes
-     *
-     * @param value the string to write (may be null or empty)
+     * Returns the current buffer capacity in bytes.
      */
-    void putString(String value);
+    int getCapacity();
 
     /**
-     * Writes UTF-8 encoded bytes directly without length prefix.
-     *
-     * @param value the string to encode (may be null or empty)
+     * Returns the current write position (number of bytes written).
      */
-    void putUtf8(String value);
-
-    // === Buffer manipulation ===
+    int getPosition();
 
     /**
      * Patches an int value at the specified offset in the buffer.
@@ -104,74 +106,58 @@ public interface QwpBufferWriter extends ArrayBufferAppender {
     void patchInt(int offset, int value);
 
     /**
-     * Skips the specified number of bytes, advancing the position.
-     * 

-     * Used when data has been written directly to the buffer via
-     * {@link #getBufferPtr()}.
-     *
-     * @param bytes number of bytes to skip
+     * Writes a float (4 bytes, little-endian).
      */
-    void skip(int bytes);
+    void putFloat(float value);
 
     /**
-     * Ensures the buffer has capacity for at least the specified
-     * additional bytes beyond the current position.
-     *
-     * @param additionalBytes number of additional bytes needed
+     * Writes a long in big-endian byte order.
      */
-    void ensureCapacity(int additionalBytes);
+    void putLongBE(long value);
 
     /**
-     * Resets the buffer for reuse, setting the position to 0.
-     * 

-     * Does not deallocate memory.
+     * Writes a short (2 bytes, little-endian).
      */
-    void reset();
-
-    // === Buffer state ===
+    void putShort(short value);
 
     /**
-     * Returns the current write position (number of bytes written).
+     * Writes a length-prefixed UTF-8 string.
+     * 

+     * Format: varint length + UTF-8 bytes
+     *
+     * @param value the string to write (may be null or empty)
      */
-    int getPosition();
+    void putString(String value);
 
     /**
-     * Returns the current buffer capacity in bytes.
+     * Writes UTF-8 encoded bytes directly without length prefix.
+     *
+     * @param value the string to encode (may be null or empty)
      */
-    int getCapacity();
+    void putUtf8(String value);
 
     /**
-     * Returns the native memory pointer to the buffer start.
+     * Writes an unsigned variable-length integer (LEB128 encoding).
      * 

-     * The returned pointer is valid until the next buffer growth operation.
-     * Use with care and only for reading completed data.
+     * Each byte contains 7 bits of data with the high bit indicating
+     * whether more bytes follow.
      */
-    long getBufferPtr();
+    void putVarint(long value);
 
-    // === Utility ===
+    /**
+     * Resets the buffer for reuse, setting the position to 0.
+     * 

+     * Does not deallocate memory.
+     */
+    void reset();
 
     /**
-     * Returns the UTF-8 encoded length of a string.
+     * Skips the specified number of bytes, advancing the position.
+     * 

+     * Used when data has been written directly to the buffer via
+     * {@link #getBufferPtr()}.
      *
-     * @param s the string (may be null)
-     * @return the number of bytes needed to encode the string as UTF-8
+     * @param bytes number of bytes to skip
      */
-    static int utf8Length(String s) {
-        if (s == null) return 0;
-        int len = 0;
-        for (int i = 0, n = s.length(); i < n; i++) {
-            char c = s.charAt(i);
-            if (c < 0x80) {
-                len++;
-            } else if (c < 0x800) {
-                len += 2;
-            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
-                i++;
-                len += 4;
-            } else {
-                len += 3;
-            }
-        }
-        return len;
-    }
+    void skip(int bytes);
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 4deb020..b2aa9a3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -50,6 +50,7 @@
 
 import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
 
+
 /**
  * ILP v4 WebSocket client sender for streaming data to QuestDB.
  * 

@@ -82,6 +83,25 @@
  *     sender.flush();
  * }
  *
+ *

+ * Fast-path API for high-throughput generators + *

+ * For maximum throughput, bypass the fluent API to avoid per-row overhead + * (no column-name hashmap lookups, no {@code checkNotClosed()}/{@code checkTableSelected()} + * per column, direct access to column buffers). Use {@link #getTableBuffer(String)}, + * {@link #getOrAddGlobalSymbol(String)}, and {@link #incrementPendingRowCount()}: + * 

+ * // Setup (once)
+ * QwpTableBuffer tableBuffer = sender.getTableBuffer("q");
+ * QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true);
+ * QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false);
+ *
+ * // Hot path (per row)
+ * colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol));
+ * colBid.addDouble(bid);
+ * tableBuffer.nextRow();
+ * sender.incrementPendingRowCount();
+ *
*/ public class QwpWebSocketSender implements Sender { @@ -219,8 +239,8 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl * @return connected sender */ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos) { + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos) { QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, @@ -518,25 +538,6 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { return this; } - // ==================== Fast-path API for high-throughput generators ==================== - // - // These methods bypass the normal fluent API to avoid per-row overhead: - // - No hashmap lookups for column names - // - No checkNotClosed()/checkTableSelected() per column - // - Direct access to column buffers - // - // Usage: - // // Setup (once) - // QwpTableBuffer tableBuffer = sender.getTableBuffer("q"); - // QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); - // QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); - // - // // Hot path (per row) - // colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol)); - // colBid.addDouble(bid); - // tableBuffer.nextRow(); - // sender.incrementPendingRowCount(); - @Override public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; @@ -573,8 +574,6 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { return this; } - // ==================== Sender interface implementation ==================== - @Override public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; @@ -614,14 +613,6 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { return this; } - /** - * Adds an INT column value to the current row. - * - * @param columnName the column name - * @param value the int value - * @return this sender for method chaining - */ - /** * Adds a FLOAT column value to the current row. * @@ -765,6 +756,14 @@ public void incrementPendingRowCount() { } } + + /** + * Adds an INT column value to the current row. + * + * @param columnName the column name + * @param value the int value + * @return this sender for method chaining + */ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); @@ -961,8 +960,6 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value return this; } - // ==================== Array methods ==================== - /** * Adds a UUID column value to the current row. * @@ -1107,8 +1104,6 @@ private void ensureConnected() { } } - // ==================== Decimal methods ==================== - private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { if (inFlightWindow != null && inFlightWindow.getLastError() == null) { inFlightWindow.fail(expectedSequence, error); @@ -1318,8 +1313,6 @@ private void sealAndSwapBuffer() { } } - // ==================== Helper methods ==================== - /** * Accumulates the current row. * Both sync and async modes buffer rows until flush (explicit or auto-flush). diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java index 1d2b89f..552e372 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java @@ -25,9 +25,9 @@ package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.QuietCloseable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.questdb.client.std.QuietCloseable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -48,24 +48,20 @@ */ public class ResponseReader implements QuietCloseable { - private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); - private static final int DEFAULT_READ_TIMEOUT_MS = 100; private static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; - + private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); private final WebSocketChannel channel; private final InFlightWindow inFlightWindow; private final Thread readerThread; - private final CountDownLatch shutdownLatch; private final WebSocketResponse response; - - // State - private volatile boolean running; - private volatile Throwable lastError; - + private final CountDownLatch shutdownLatch; // Statistics private final AtomicLong totalAcks = new AtomicLong(0); private final AtomicLong totalErrors = new AtomicLong(0); + private volatile Throwable lastError; + // State + private volatile boolean running; /** * Creates a new response reader. @@ -96,6 +92,26 @@ public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) { LOG.info("Response reader started"); } + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing response reader"); + + running = false; + + // Wait for reader thread to finish + try { + shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + } + /** * Returns the last error that occurred, or null if no error. */ @@ -103,13 +119,6 @@ public Throwable getLastError() { return lastError; } - /** - * Returns true if the reader is still running. - */ - public boolean isRunning() { - return running; - } - /** * Returns total successful acknowledgments received. */ @@ -124,28 +133,13 @@ public long getTotalErrors() { return totalErrors.get(); } - @Override - public void close() { - if (!running) { - return; - } - - LOG.info("Closing response reader"); - - running = false; - - // Wait for reader thread to finish - try { - shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + /** + * Returns true if the reader is still running. + */ + public boolean isRunning() { + return running; } - // ==================== Reader Thread ==================== - /** * Main read loop that processes incoming WebSocket frames. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java index 415ee4b..da64ca9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java @@ -24,12 +24,12 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; -import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Rnd; @@ -68,45 +68,40 @@ public class WebSocketChannel implements QuietCloseable { private static final int DEFAULT_BUFFER_SIZE = 65536; private static final int MAX_FRAME_HEADER_SIZE = 14; // 2 + 8 + 4 (header + extended len + mask) - + // Frame parser (reused) + private final WebSocketFrameParser frameParser; + // Temporary byte array for handshake (allocated once) + private final byte[] handshakeBuffer = new byte[4096]; // Connection state private final String host; - private final int port; private final String path; + private final int port; + // Random for mask key generation + private final Rnd rnd; private final boolean tlsEnabled; private final boolean tlsValidationEnabled; - - // Socket I/O - private Socket socket; + private boolean closed; + // Timeouts + private int connectTimeoutMs = 10_000; + // State + private boolean connected; private InputStream in; private OutputStream out; - - // Pre-allocated send buffer (native memory) - private long sendBufferPtr; - private int sendBufferSize; - + private byte[] readTempBuffer; + private int readTimeoutMs = 30_000; + private int recvBufferPos; // Write position // Pre-allocated receive buffer (native memory) private long recvBufferPtr; - private int recvBufferSize; - private int recvBufferPos; // Write position private int recvBufferReadPos; // Read position - - // Frame parser (reused) - private final WebSocketFrameParser frameParser; - - // Random for mask key generation - private final Rnd rnd; - - // Timeouts - private int connectTimeoutMs = 10_000; - private int readTimeoutMs = 30_000; - - // State - private boolean connected; - private boolean closed; - - // Temporary byte array for handshake (allocated once) - private final byte[] handshakeBuffer = new byte[4096]; + private int recvBufferSize; + // Pre-allocated send buffer (native memory) + private long sendBufferPtr; + private int sendBufferSize; + // Socket I/O + private Socket socket; + // Separate temp buffers for read and write to avoid race conditions + // between send queue thread and response reader thread + private byte[] writeTempBuffer; public WebSocketChannel(String url, boolean tlsEnabled) { this(url, tlsEnabled, true); @@ -162,19 +157,35 @@ public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEna } /** - * Sets the connection timeout. + * Sends a close frame and closes the connection. */ - public WebSocketChannel setConnectTimeout(int timeoutMs) { - this.connectTimeoutMs = timeoutMs; - return this; - } + @Override + public void close() { + if (closed) { + return; + } + closed = true; - /** - * Sets the read timeout. - */ - public WebSocketChannel setReadTimeout(int timeoutMs) { - this.readTimeoutMs = timeoutMs; - return this; + try { + if (connected) { + // Send close frame + sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); + } + } catch (Exception e) { + // Ignore errors during close + } + + closeQuietly(); + + // Free native memory + if (sendBufferPtr != 0) { + Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); + sendBufferPtr = 0; + } + if (recvBufferPtr != 0) { + Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); + recvBufferPtr = 0; + } } /** @@ -210,31 +221,15 @@ public void connect() { } } - /** - * Sends binary data as a WebSocket binary frame. - * The data is read from native memory at the given pointer. - * - * @param dataPtr pointer to the data - * @param length length of data in bytes - */ - public void sendBinary(long dataPtr, int length) { - ensureConnected(); - sendFrame(WebSocketOpcode.BINARY, dataPtr, length); - } - - /** - * Sends a ping frame. - */ - public void sendPing() { - ensureConnected(); - sendFrame(WebSocketOpcode.PING, 0, 0); + public boolean isConnected() { + return connected && !closed; } /** * Receives and processes incoming frames. * Handles ping/pong automatically. * - * @param handler callback for received binary messages (may be null) + * @param handler callback for received binary messages (may be null) * @param timeoutMs read timeout in milliseconds * @return true if a frame was received, false on timeout */ @@ -256,43 +251,100 @@ public boolean receiveFrame(ResponseHandler handler, int timeoutMs) { } /** - * Sends a close frame and closes the connection. + * Sends binary data as a WebSocket binary frame. + * The data is read from native memory at the given pointer. + * + * @param dataPtr pointer to the data + * @param length length of data in bytes */ - @Override - public void close() { - if (closed) { - return; + public void sendBinary(long dataPtr, int length) { + ensureConnected(); + sendFrame(WebSocketOpcode.BINARY, dataPtr, length); + } + + /** + * Sends a ping frame. + */ + public void sendPing() { + ensureConnected(); + sendFrame(WebSocketOpcode.PING, 0, 0); + } + + /** + * Sets the connection timeout. + */ + public WebSocketChannel setConnectTimeout(int timeoutMs) { + this.connectTimeoutMs = timeoutMs; + return this; + } + + /** + * Sets the read timeout. + */ + public WebSocketChannel setReadTimeout(int timeoutMs) { + this.readTimeoutMs = timeoutMs; + return this; + } + + private void closeQuietly() { + connected = false; + if (socket != null) { + try { + socket.close(); + } catch (IOException e) { + // Ignore + } + socket = null; } - closed = true; + in = null; + out = null; + } + private SocketFactory createSslSocketFactory() { try { - if (connected) { - // Send close frame - sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); + if (!tlsValidationEnabled) { + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new TrustManager[]{new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] certs, String t) { + } + + public void checkServerTrusted(X509Certificate[] certs, String t) { + } + + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }}, new SecureRandom()); + return sslContext.getSocketFactory(); } + return SSLSocketFactory.getDefault(); } catch (Exception e) { - // Ignore errors during close + throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); } + } - closeQuietly(); - - // Free native memory - if (sendBufferPtr != 0) { - Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); - sendBufferPtr = 0; + private boolean doReceiveFrame(ResponseHandler handler) throws IOException { + // First, try to parse any data already in the buffer + // This handles the case where multiple frames arrived in a single TCP read + if (recvBufferPos > recvBufferReadPos) { + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + // result == null means we need more data, continue to read } - if (recvBufferPtr != 0) { - Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); - recvBufferPtr = 0; + + // Read more data into receive buffer + int bytesRead = readFromSocket(); + if (bytesRead <= 0) { + return false; } - } - public boolean isConnected() { - return connected && !closed; + // Try parsing again with the new data + Boolean result = tryParseFrame(handler); + return result != null && result; } - // ==================== Private methods ==================== - private void ensureConnected() { if (closed) { throw new LineSenderException("WebSocket channel is closed"); @@ -302,21 +354,26 @@ private void ensureConnected() { } } - private SocketFactory createSslSocketFactory() { - try { - if (!tlsValidationEnabled) { - SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, new TrustManager[]{new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] certs, String t) {} - public void checkServerTrusted(X509Certificate[] certs, String t) {} - public X509Certificate[] getAcceptedIssuers() { return null; } - }}, new SecureRandom()); - return sslContext.getSocketFactory(); - } - return SSLSocketFactory.getDefault(); - } catch (Exception e) { - throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); + private void ensureSendBufferSize(int required) { + if (required > sendBufferSize) { + int newSize = Math.max(required, sendBufferSize * 2); + sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + sendBufferSize = newSize; + } + } + + private byte[] getReadTempBuffer(int minSize) { + if (readTempBuffer == null || readTempBuffer.length < minSize) { + readTempBuffer = new byte[Math.max(minSize, 8192)]; + } + return readTempBuffer; + } + + private byte[] getWriteTempBuffer(int minSize) { + if (writeTempBuffer == null || writeTempBuffer.length < minSize) { + writeTempBuffer = new byte[Math.max(minSize, 8192)]; } + return writeTempBuffer; } private void performHandshake() throws IOException { @@ -364,6 +421,28 @@ private void performHandshake() throws IOException { } } + private int readFromSocket() throws IOException { + // Ensure space in receive buffer + int available = recvBufferSize - recvBufferPos; + if (available < 1024) { + // Grow buffer + int newSize = recvBufferSize * 2; + recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + recvBufferSize = newSize; + available = recvBufferSize - recvBufferPos; + } + + // Read into temp array then copy to native buffer + // Use separate read buffer to avoid race with write thread + byte[] temp = getReadTempBuffer(available); + int bytesRead = in.read(temp, 0, available); + if (bytesRead > 0) { + Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead); + recvBufferPos += bytesRead; + } + return bytesRead; + } + private int readHttpResponse() throws IOException { int pos = 0; int consecutiveCrLf = 0; @@ -378,9 +457,9 @@ private int readHttpResponse() throws IOException { // Look for \r\n\r\n if (b == '\r' || b == '\n') { if ((consecutiveCrLf == 0 && b == '\r') || - (consecutiveCrLf == 1 && b == '\n') || - (consecutiveCrLf == 2 && b == '\r') || - (consecutiveCrLf == 3 && b == '\n')) { + (consecutiveCrLf == 1 && b == '\n') || + (consecutiveCrLf == 2 && b == '\r') || + (consecutiveCrLf == 3 && b == '\n')) { consecutiveCrLf++; if (consecutiveCrLf == 4) { return pos; @@ -395,36 +474,6 @@ private int readHttpResponse() throws IOException { throw new IOException("HTTP response too large"); } - private void sendFrame(int opcode, long payloadPtr, int payloadLen) { - // Generate mask key - int maskKey = rnd.nextInt(); - - // Calculate required buffer size - int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); - int frameSize = headerSize + payloadLen; - - // Ensure buffer is large enough - ensureSendBufferSize(frameSize); - - // Write frame header with mask - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, opcode, payloadLen, maskKey); - - // Copy payload to buffer after header - if (payloadLen > 0) { - Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); - // Mask the payload in place - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); - } - - // Send frame - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); - } - } - private void sendCloseFrame(int code, String reason) { int maskKey = rnd.nextInt(); @@ -465,30 +514,61 @@ private void sendCloseFrame(int code, String reason) { } } - private boolean doReceiveFrame(ResponseHandler handler) throws IOException { - // First, try to parse any data already in the buffer - // This handles the case where multiple frames arrived in a single TCP read - if (recvBufferPos > recvBufferReadPos) { - Boolean result = tryParseFrame(handler); - if (result != null) { - return result; - } - // result == null means we need more data, continue to read + private void sendFrame(int opcode, long payloadPtr, int payloadLen) { + // Generate mask key + int maskKey = rnd.nextInt(); + + // Calculate required buffer size + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int frameSize = headerSize + payloadLen; + + // Ensure buffer is large enough + ensureSendBufferSize(frameSize); + + // Write frame header with mask + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, opcode, payloadLen, maskKey); + + // Copy payload to buffer after header + if (payloadLen > 0) { + Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); + // Mask the payload in place + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); } - // Read more data into receive buffer - int bytesRead = readFromSocket(); - if (bytesRead <= 0) { - return false; + // Send frame + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); } + } - // Try parsing again with the new data - Boolean result = tryParseFrame(handler); - return result != null && result; + private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { + int maskKey = rnd.nextInt(); + int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); + int frameSize = headerSize + pingPayloadLen; + + ensureSendBufferSize(frameSize); + + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); + + if (pingPayloadLen > 0) { + Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); + } + + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + // Ignore pong send errors + } } /** * Tries to parse a frame from the receive buffer. + * * @return true if frame processed, false if error, null if need more data */ private Boolean tryParseFrame(ResponseHandler handler) throws IOException { @@ -561,36 +641,6 @@ private Boolean tryParseFrame(ResponseHandler handler) throws IOException { return false; } - private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { - int maskKey = rnd.nextInt(); - int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); - int frameSize = headerSize + pingPayloadLen; - - ensureSendBufferSize(frameSize); - - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); - - if (pingPayloadLen > 0) { - Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); - } - - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - // Ignore pong send errors - } - } - - private void ensureSendBufferSize(int required) { - if (required > sendBufferSize) { - int newSize = Math.max(required, sendBufferSize * 2); - sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - sendBufferSize = newSize; - } - } - private void writeToSocket(long ptr, int len) throws IOException { // Copy to temp array for socket write (unavoidable with OutputStream) // Use separate write buffer to avoid race with read thread @@ -600,66 +650,12 @@ private void writeToSocket(long ptr, int len) throws IOException { out.flush(); } - private int readFromSocket() throws IOException { - // Ensure space in receive buffer - int available = recvBufferSize - recvBufferPos; - if (available < 1024) { - // Grow buffer - int newSize = recvBufferSize * 2; - recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - recvBufferSize = newSize; - available = recvBufferSize - recvBufferPos; - } - - // Read into temp array then copy to native buffer - // Use separate read buffer to avoid race with write thread - byte[] temp = getReadTempBuffer(available); - int bytesRead = in.read(temp, 0, available); - if (bytesRead > 0) { - Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead); - recvBufferPos += bytesRead; - } - return bytesRead; - } - - // Separate temp buffers for read and write to avoid race conditions - // between send queue thread and response reader thread - private byte[] writeTempBuffer; - private byte[] readTempBuffer; - - private byte[] getWriteTempBuffer(int minSize) { - if (writeTempBuffer == null || writeTempBuffer.length < minSize) { - writeTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return writeTempBuffer; - } - - private byte[] getReadTempBuffer(int minSize) { - if (readTempBuffer == null || readTempBuffer.length < minSize) { - readTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return readTempBuffer; - } - - private void closeQuietly() { - connected = false; - if (socket != null) { - try { - socket.close(); - } catch (IOException e) { - // Ignore - } - socket = null; - } - in = null; - out = null; - } - /** * Callback interface for received WebSocket messages. */ public interface ResponseHandler { void onBinaryMessage(long payload, int length); + void onClose(int code, String reason); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 35a5f77..e1c1e6b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -54,22 +54,20 @@ */ public class WebSocketResponse { + public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; + public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length + // Minimum response size: status (1) + sequence (8) + public static final int MIN_RESPONSE_SIZE = 9; + public static final byte STATUS_INTERNAL_ERROR = (byte) 255; // Status codes public static final byte STATUS_OK = 0; public static final byte STATUS_PARSE_ERROR = 1; public static final byte STATUS_SCHEMA_ERROR = 2; - public static final byte STATUS_WRITE_ERROR = 3; public static final byte STATUS_SECURITY_ERROR = 4; - public static final byte STATUS_INTERNAL_ERROR = (byte) 255; - - // Minimum response size: status (1) + sequence (8) - public static final int MIN_RESPONSE_SIZE = 9; - public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length - public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; - - private byte status; - private long sequence; + public static final byte STATUS_WRITE_ERROR = 3; private String errorMessage; + private long sequence; + private byte status; public WebSocketResponse() { this.status = STATUS_OK; @@ -77,16 +75,6 @@ public WebSocketResponse() { this.errorMessage = null; } - /** - * Creates a success response. - */ - public static WebSocketResponse success(long sequence) { - WebSocketResponse response = new WebSocketResponse(); - response.status = STATUS_OK; - response.sequence = sequence; - return response; - } - /** * Creates an error response. */ @@ -130,17 +118,20 @@ public static boolean isStructurallyValid(long ptr, int length) { } /** - * Returns true if this is a success response. + * Creates a success response. */ - public boolean isSuccess() { - return status == STATUS_OK; + public static WebSocketResponse success(long sequence) { + WebSocketResponse response = new WebSocketResponse(); + response.status = STATUS_OK; + response.sequence = sequence; + return response; } /** - * Returns the status code. + * Returns the error message, or null for success responses. */ - public byte getStatus() { - return status; + public String getErrorMessage() { + return errorMessage; } /** @@ -151,10 +142,10 @@ public long getSequence() { } /** - * Returns the error message, or null for success responses. + * Returns the status code. */ - public String getErrorMessage() { - return errorMessage; + public byte getStatus() { + return status; } /** @@ -180,52 +171,10 @@ public String getStatusName() { } /** - * Calculates the serialized size of this response. - */ - public int serializedSize() { - int size = MIN_RESPONSE_SIZE; - if (errorMessage != null && !errorMessage.isEmpty()) { - byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); - int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); - size += 2 + msgLen; // 2 bytes for length prefix - } - return size; - } - - /** - * Writes this response to native memory. - * - * @param ptr destination address - * @return number of bytes written + * Returns true if this is a success response. */ - public int writeTo(long ptr) { - int offset = 0; - - // Status (1 byte) - Unsafe.getUnsafe().putByte(ptr + offset, status); - offset += 1; - - // Sequence (8 bytes, little-endian) - Unsafe.getUnsafe().putLong(ptr + offset, sequence); - offset += 8; - - // Error message (if any) - if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) { - byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); - int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); - - // Length prefix (2 bytes, little-endian) - Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen); - offset += 2; - - // Message bytes - for (int i = 0; i < msgLen; i++) { - Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]); - } - offset += msgLen; - } - - return offset; + public boolean isSuccess() { + return status == STATUS_OK; } /** @@ -270,6 +219,19 @@ public boolean readFrom(long ptr, int length) { return true; } + /** + * Calculates the serialized size of this response. + */ + public int serializedSize() { + int size = MIN_RESPONSE_SIZE; + if (errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + size += 2 + msgLen; // 2 bytes for length prefix + } + return size; + } + @Override public String toString() { if (isSuccess()) { @@ -279,4 +241,40 @@ public String toString() { ", error=" + errorMessage + "}"; } } + + /** + * Writes this response to native memory. + * + * @param ptr destination address + * @return number of bytes written + */ + public int writeTo(long ptr) { + int offset = 0; + + // Status (1 byte) + Unsafe.getUnsafe().putByte(ptr + offset, status); + offset += 1; + + // Sequence (8 bytes, little-endian) + Unsafe.getUnsafe().putLong(ptr + offset, sequence); + offset += 8; + + // Error message (if any) + if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + + // Length prefix (2 bytes, little-endian) + Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen); + offset += 2; + + // Message bytes + for (int i = 0; i < msgLen; i++) { + Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]); + } + offset += msgLen; + } + + return offset; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index e47c1d8..9a8a8bb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -27,10 +27,10 @@ import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.line.LineSenderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -62,90 +62,50 @@ */ public class WebSocketSendQueue implements QuietCloseable { - private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); - + public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; // Default configuration public static final int DEFAULT_QUEUE_CAPACITY = 16; - public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; - - // Single pending buffer slot (double-buffering means at most 1 item in queue) - // Zero allocation - just a volatile reference handoff - private volatile MicrobatchBuffer pendingBuffer; - + private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); // The WebSocket client for I/O (single-threaded access only) private final WebSocketClient client; - + // Configuration + private final long enqueueTimeoutMs; // Optional InFlightWindow for tracking sent batches awaiting ACK @Nullable private final InFlightWindow inFlightWindow; // The I/O thread for async send/receive private final Thread ioThread; - - // Running state - private volatile boolean running; - private volatile boolean shuttingDown; - - // Synchronization for flush/close - private final CountDownLatch shutdownLatch; - - // Error handling - private volatile Throwable lastError; - - // Statistics - sending - private final AtomicLong totalBatchesSent = new AtomicLong(0); - private final AtomicLong totalBytesSent = new AtomicLong(0); - - // Statistics - receiving - private final AtomicLong totalAcks = new AtomicLong(0); - private final AtomicLong totalErrors = new AtomicLong(0); - // Counter for batches currently being processed by the I/O thread // This tracks batches that have been dequeued but not yet fully sent private final AtomicInteger processingCount = new AtomicInteger(0); - // Lock for all coordination between user thread and I/O thread. // Used for: queue poll + processingCount increment atomicity, // flush() waiting, I/O thread waiting when idle. private final Object processingLock = new Object(); - - // Batch sequence counter (must match server's messageSequence) - private long nextBatchSequence = 0; - // Response parsing private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); - - // Configuration - private final long enqueueTimeoutMs; + // Synchronization for flush/close + private final CountDownLatch shutdownLatch; private final long shutdownTimeoutMs; - - // ==================== Pending Buffer Operations (zero allocation) ==================== - - private boolean offerPending(MicrobatchBuffer buffer) { - if (pendingBuffer != null) { - return false; // slot occupied - } - pendingBuffer = buffer; - return true; - } - - private MicrobatchBuffer pollPending() { - MicrobatchBuffer buffer = pendingBuffer; - if (buffer != null) { - pendingBuffer = null; - } - return buffer; - } - - private boolean isPendingEmpty() { - return pendingBuffer == null; - } - - private int getPendingSize() { - return pendingBuffer == null ? 0 : 1; - } + // Statistics - receiving + private final AtomicLong totalAcks = new AtomicLong(0); + // Statistics - sending + private final AtomicLong totalBatchesSent = new AtomicLong(0); + private final AtomicLong totalBytesSent = new AtomicLong(0); + private final AtomicLong totalErrors = new AtomicLong(0); + // Error handling + private volatile Throwable lastError; + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + // Single pending buffer slot (double-buffering means at most 1 item in queue) + // Zero allocation - just a volatile reference handoff + private volatile MicrobatchBuffer pendingBuffer; + // Running state + private volatile boolean running; + private volatile boolean shuttingDown; /** * Creates a new send queue with default configuration. @@ -200,6 +160,61 @@ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFli LOG.info("WebSocket I/O thread started [capacity={}]", queueCapacity); } + /** + * Closes the send queue gracefully. + *

+ * This method: + * 1. Stops accepting new batches + * 2. Waits for pending batches to be sent + * 3. Stops the I/O thread + *

+ * Note: This does NOT close the WebSocket channel - that's the caller's responsibility. + */ + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize()); + + // Signal shutdown + shuttingDown = true; + + // Wait for pending batches to be sent + long startTime = System.currentTimeMillis(); + while (!isPendingEmpty()) { + if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { + LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); + break; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + // Stop the I/O thread + running = false; + + // Wake up I/O thread if it's blocked on processingLock.wait() + synchronized (processingLock) { + processingLock.notifyAll(); + } + ioThread.interrupt(); + + // Wait for I/O thread to finish + try { + shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get()); + } + /** * Enqueues a sealed buffer for sending. *

@@ -301,24 +316,24 @@ public void flush() { } /** - * Returns the number of batches waiting to be sent. + * Returns the last error that occurred in the I/O thread, or null if no error. */ - public int getPendingCount() { - return getPendingSize(); + public Throwable getLastError() { + return lastError; } /** - * Returns true if the queue is empty. + * Returns the number of batches waiting to be sent. */ - public boolean isEmpty() { - return isPendingEmpty(); + public int getPendingCount() { + return getPendingSize(); } /** - * Returns true if the queue is still running. + * Returns total successful acknowledgments received. */ - public boolean isRunning() { - return running && !shuttingDown; + public long getTotalAcks() { + return totalAcks.get(); } /** @@ -336,79 +351,76 @@ public long getTotalBytesSent() { } /** - * Returns the last error that occurred in the I/O thread, or null if no error. + * Returns total error responses received. */ - public Throwable getLastError() { - return lastError; + public long getTotalErrors() { + return totalErrors.get(); } /** - * Closes the send queue gracefully. - *

- * This method: - * 1. Stops accepting new batches - * 2. Waits for pending batches to be sent - * 3. Stops the I/O thread - *

- * Note: This does NOT close the WebSocket channel - that's the caller's responsibility. + * Returns true if the queue is empty. */ - @Override - public void close() { - if (!running) { - return; - } + public boolean isEmpty() { + return isPendingEmpty(); + } - LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize()); + /** + * Returns true if the queue is still running. + */ + public boolean isRunning() { + return running && !shuttingDown; + } - // Signal shutdown - shuttingDown = true; + /** + * Checks if an error occurred in the I/O thread and throws if so. + */ + private void checkError() { + Throwable error = lastError; + if (error != null) { + throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error); + } + } - // Wait for pending batches to be sent - long startTime = System.currentTimeMillis(); - while (!isPendingEmpty()) { - if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { - LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); - break; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } + /** + * Computes the current I/O state based on queue and in-flight status. + */ + private IoState computeState(boolean hasInFlight) { + if (!isPendingEmpty()) { + return IoState.ACTIVE; + } else if (hasInFlight) { + return IoState.DRAINING; + } else { + return IoState.IDLE; } + } - // Stop the I/O thread + private void failTransport(LineSenderException error) { + Throwable rootError = lastError; + if (rootError == null) { + lastError = error; + rootError = error; + } running = false; - - // Wake up I/O thread if it's blocked on processingLock.wait() + shuttingDown = true; + if (inFlightWindow != null) { + inFlightWindow.failAll(rootError); + } synchronized (processingLock) { + MicrobatchBuffer dropped = pollPending(); + if (dropped != null) { + if (dropped.isSealed()) { + dropped.markSending(); + } + if (dropped.isSending()) { + dropped.markRecycled(); + } + } processingLock.notifyAll(); } - ioThread.interrupt(); - - // Wait for I/O thread to finish - try { - shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get()); } - // ==================== I/O Thread ==================== - - /** - * I/O loop states for the state machine. - *

    - *
  • IDLE: queue empty, no in-flight batches - can block waiting for work
    • - *
  • ACTIVE: have batches to send - non-blocking loop
    • - *
  • DRAINING: queue empty but ACKs pending - poll for ACKs, short wait
    • - *
- */ - private enum IoState { - IDLE, ACTIVE, DRAINING + private int getPendingSize() { + return pendingBuffer == null ? 0 : 1; } /** @@ -495,31 +507,24 @@ private void ioLoop() { } } - /** - * Computes the current I/O state based on queue and in-flight status. - */ - private IoState computeState(boolean hasInFlight) { - if (!isPendingEmpty()) { - return IoState.ACTIVE; - } else if (hasInFlight) { - return IoState.DRAINING; - } else { - return IoState.IDLE; + private boolean isPendingEmpty() { + return pendingBuffer == null; + } + + private boolean offerPending(MicrobatchBuffer buffer) { + if (pendingBuffer != null) { + return false; // slot occupied } + pendingBuffer = buffer; + return true; } - /** - * Tries to receive ACKs from the server (non-blocking). - */ - private void tryReceiveAcks() { - try { - client.tryReceiveFrame(responseHandler); - } catch (Exception e) { - if (running) { - LOG.error("Error receiving response: {}", e.getMessage()); - failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); - } + private MicrobatchBuffer pollPending() { + MicrobatchBuffer buffer = pendingBuffer; + if (buffer != null) { + pendingBuffer = null; } + return buffer; } /** @@ -582,56 +587,31 @@ private void sendBatch(MicrobatchBuffer batch) { } /** - * Checks if an error occurred in the I/O thread and throws if so. + * Tries to receive ACKs from the server (non-blocking). */ - private void checkError() { - Throwable error = lastError; - if (error != null) { - throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error); - } - } - - private void failTransport(LineSenderException error) { - Throwable rootError = lastError; - if (rootError == null) { - lastError = error; - rootError = error; - } - running = false; - shuttingDown = true; - if (inFlightWindow != null) { - inFlightWindow.failAll(rootError); - } - synchronized (processingLock) { - MicrobatchBuffer dropped = pollPending(); - if (dropped != null) { - if (dropped.isSealed()) { - dropped.markSending(); - } - if (dropped.isSending()) { - dropped.markRecycled(); - } + private void tryReceiveAcks() { + try { + client.tryReceiveFrame(responseHandler); + } catch (Exception e) { + if (running) { + LOG.error("Error receiving response: {}", e.getMessage()); + failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); } - processingLock.notifyAll(); } } /** - * Returns total successful acknowledgments received. - */ - public long getTotalAcks() { - return totalAcks.get(); - } - - /** - * Returns total error responses received. + * I/O loop states for the state machine. + *
    + *
  • IDLE: queue empty, no in-flight batches - can block waiting for work
    • + *
  • ACTIVE: have batches to send - non-blocking loop
    • + *
  • DRAINING: queue empty but ACKs pending - poll for ACKs, short wait
    • + *
*/ - public long getTotalErrors() { - return totalErrors.get(); + private enum IoState { + IDLE, ACTIVE, DRAINING } - // ==================== Response Handler ==================== - /** * Handler for received WebSocket frames (ACKs from server). */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index 5830ea7..b73d88c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -40,10 +40,9 @@ public class OffHeapAppendMemory implements QuietCloseable { private static final int DEFAULT_INITIAL_CAPACITY = 128; - - private long pageAddress; private long appendAddress; private long capacity; + private long pageAddress; public OffHeapAppendMemory() { this(DEFAULT_INITIAL_CAPACITY); @@ -55,20 +54,6 @@ public OffHeapAppendMemory(long initialCapacity) { this.appendAddress = pageAddress; } - /** - * Returns the append offset (number of bytes written). - */ - public long getAppendOffset() { - return appendAddress - pageAddress; - } - - /** - * Returns the base address of the buffer. - */ - public long pageAddress() { - return pageAddress; - } - /** * Returns the address at the given byte offset from the start. */ @@ -76,11 +61,21 @@ public long addressOf(long offset) { return pageAddress + offset; } + @Override + public void close() { + if (pageAddress != 0) { + Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS); + pageAddress = 0; + appendAddress = 0; + capacity = 0; + } + } + /** - * Resets the append position to 0 without freeing memory. + * Returns the append offset (number of bytes written). */ - public void truncate() { - appendAddress = pageAddress; + public long getAppendOffset() { + return appendAddress - pageAddress; } /** @@ -91,20 +86,33 @@ public void jumpTo(long offset) { appendAddress = pageAddress + offset; } + /** + * Returns the base address of the buffer. + */ + public long pageAddress() { + return pageAddress; + } + + public void putBoolean(boolean value) { + putByte(value ? (byte) 1 : (byte) 0); + } + public void putByte(byte value) { ensureCapacity(1); Unsafe.getUnsafe().putByte(appendAddress, value); appendAddress++; } - public void putBoolean(boolean value) { - putByte(value ? (byte) 1 : (byte) 0); + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(appendAddress, value); + appendAddress += 8; } - public void putShort(short value) { - ensureCapacity(2); - Unsafe.getUnsafe().putShort(appendAddress, value); - appendAddress += 2; + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(appendAddress, value); + appendAddress += 4; } public void putInt(int value) { @@ -119,16 +127,10 @@ public void putLong(long value) { appendAddress += 8; } - public void putFloat(float value) { - ensureCapacity(4); - Unsafe.getUnsafe().putFloat(appendAddress, value); - appendAddress += 4; - } - - public void putDouble(double value) { - ensureCapacity(8); - Unsafe.getUnsafe().putDouble(appendAddress, value); - appendAddress += 8; + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(appendAddress, value); + appendAddress += 2; } /** @@ -171,14 +173,11 @@ public void skip(long bytes) { appendAddress += bytes; } - @Override - public void close() { - if (pageAddress != 0) { - Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS); - pageAddress = 0; - appendAddress = 0; - capacity = 0; - } + /** + * Resets the append position to 0 without freeing memory. + */ + public void truncate() { + appendAddress = pageAddress; } private void ensureCapacity(long needed) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index af30761..3f18d2d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -48,14 +48,13 @@ */ public class QwpBitWriter { - private long startAddress; - private long currentAddress; - private long endAddress; - // Buffer for accumulating bits before writing private long bitBuffer; // Number of bits currently in the buffer (0-63) private int bitsInBuffer; + private long currentAddress; + private long endAddress; + private long startAddress; /** * Creates a new bit writer. Call {@link #reset} before use. @@ -64,17 +63,54 @@ public QwpBitWriter() { } /** - * Resets the writer to write to the specified memory region. + * Aligns the writer to the next byte boundary by padding with zeros. + * If already byte-aligned, this is a no-op. + */ + public void alignToByte() { + if (bitsInBuffer > 0) { + flush(); + } + } + + /** + * Finishes writing and returns the number of bytes written since reset. + *

+ * This method flushes any remaining bits and returns the total byte count. * - * @param address the starting address - * @param capacity the maximum number of bytes to write + * @return bytes written since reset */ - public void reset(long address, long capacity) { - this.startAddress = address; - this.currentAddress = address; - this.endAddress = address + capacity; - this.bitBuffer = 0; - this.bitsInBuffer = 0; + public int finish() { + flush(); + return (int) (currentAddress - startAddress); + } + + /** + * Flushes any remaining bits in the buffer to memory. + *

+ * If there are partial bits (less than 8), they are written as the last byte + * with the remaining high bits set to zero. + *

+ * Must be called before reading the output or getting the final position. + */ + public void flush() { + if (bitsInBuffer > 0) { + if (currentAddress >= endAddress) { + throw new LineSenderException("QwpBitWriter buffer overflow"); + } + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + bitBuffer = 0; + bitsInBuffer = 0; + } + } + + /** + * Returns the number of bits remaining in the partial byte buffer. + * This is 0 after a flush or when aligned on a byte boundary. + * + * @return bits in buffer (0-7) + */ + public int getBitsInBuffer() { + return bitsInBuffer; } /** @@ -96,6 +132,20 @@ public long getTotalBitsWritten() { return (currentAddress - startAddress) * 8L + bitsInBuffer; } + /** + * Resets the writer to write to the specified memory region. + * + * @param address the starting address + * @param capacity the maximum number of bytes to write + */ + public void reset(long address, long capacity) { + this.startAddress = address; + this.currentAddress = address; + this.endAddress = address + capacity; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + } + /** * Writes a single bit. * @@ -150,68 +200,6 @@ public void writeBits(long value, int numBits) { } } - /** - * Writes a signed value using two's complement representation. - * - * @param value the signed value - * @param numBits number of bits to use for the representation - */ - public void writeSigned(long value, int numBits) { - // Two's complement is automatic in Java for the bit pattern - writeBits(value, numBits); - } - - /** - * Flushes any remaining bits in the buffer to memory. - *

- * If there are partial bits (less than 8), they are written as the last byte - * with the remaining high bits set to zero. - *

- * Must be called before reading the output or getting the final position. - */ - public void flush() { - if (bitsInBuffer > 0) { - if (currentAddress >= endAddress) { - throw new LineSenderException("QwpBitWriter buffer overflow"); - } - Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); - bitBuffer = 0; - bitsInBuffer = 0; - } - } - - /** - * Finishes writing and returns the number of bytes written since reset. - *

- * This method flushes any remaining bits and returns the total byte count. - * - * @return bytes written since reset - */ - public int finish() { - flush(); - return (int) (currentAddress - startAddress); - } - - /** - * Returns the number of bits remaining in the partial byte buffer. - * This is 0 after a flush or when aligned on a byte boundary. - * - * @return bits in buffer (0-7) - */ - public int getBitsInBuffer() { - return bitsInBuffer; - } - - /** - * Aligns the writer to the next byte boundary by padding with zeros. - * If already byte-aligned, this is a no-op. - */ - public void alignToByte() { - if (bitsInBuffer > 0) { - flush(); - } - } - /** * Writes a complete byte, ensuring byte alignment first. * @@ -252,4 +240,15 @@ public void writeLong(long value) { Unsafe.getUnsafe().putLong(currentAddress, value); currentAddress += 8; } + + /** + * Writes a signed value using two's complement representation. + * + * @param value the signed value + * @param numBits number of bits to use for the representation + */ + public void writeSigned(long value, int numBits) { + // Two's complement is automatic in Java for the bit pattern + writeBits(value, numBits); + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index a7257dd..f59a009 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -24,7 +24,8 @@ package io.questdb.client.cutlass.qwp.protocol; -import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_BOOLEAN; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_CHAR; /** * Represents a column definition in an ILP v4 schema. @@ -33,8 +34,8 @@ */ public final class QwpColumnDef { private final String name; - private final byte typeCode; private final boolean nullable; + private final byte typeCode; /** * Creates a column definition. @@ -62,6 +63,25 @@ public QwpColumnDef(String name, byte typeCode, boolean nullable) { this.nullable = nullable; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QwpColumnDef that = (QwpColumnDef) o; + return typeCode == that.typeCode && + nullable == that.nullable && + name.equals(that.name); + } + + /** + * Gets the fixed width in bytes for fixed-width types. + * + * @return width in bytes, or -1 for variable-width types + */ + public int getFixedWidth() { + return QwpConstants.getFixedTypeSize(typeCode); + } + /** * Gets the column name. */ @@ -78,6 +98,13 @@ public byte getTypeCode() { return typeCode; } + /** + * Gets the type name for display purposes. + */ + public String getTypeName() { + return QwpConstants.getTypeName(typeCode); + } + /** * Gets the wire type code (with nullable flag if applicable). * @@ -87,11 +114,12 @@ public byte getWireTypeCode() { return nullable ? (byte) (typeCode | 0x80) : typeCode; } - /** - * Returns true if this column is nullable. - */ - public boolean isNullable() { - return nullable; + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + typeCode; + result = 31 * result + (nullable ? 1 : 0); + return result; } /** @@ -102,19 +130,20 @@ public boolean isFixedWidth() { } /** - * Gets the fixed width in bytes for fixed-width types. - * - * @return width in bytes, or -1 for variable-width types + * Returns true if this column is nullable. */ - public int getFixedWidth() { - return QwpConstants.getFixedTypeSize(typeCode); + public boolean isNullable() { + return nullable; } - /** - * Gets the type name for display purposes. - */ - public String getTypeName() { - return QwpConstants.getTypeName(typeCode); + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name).append(':').append(getTypeName()); + if (nullable) { + sb.append('?'); + } + return sb.toString(); } /** @@ -132,32 +161,4 @@ public void validate() { ); } } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - QwpColumnDef that = (QwpColumnDef) o; - return typeCode == that.typeCode && - nullable == that.nullable && - name.equals(that.name); - } - - @Override - public int hashCode() { - int result = name.hashCode(); - result = 31 * result + typeCode; - result = 31 * result + (nullable ? 1 : 0); - return result; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(name).append(':').append(getTypeName()); - if (nullable) { - sb.append('?'); - } - return sb.toString(); - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 2a40a35..34f9b2d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -29,68 +29,48 @@ */ public final class QwpConstants { - // ==================== Magic Bytes ==================== - - /** - * Magic bytes for ILP v4 message: "ILP4" (ASCII). - */ - public static final int MAGIC_MESSAGE = 0x34504C49; // "ILP4" in little-endian - /** - * Magic bytes for capability request: "ILP?" (ASCII). + * Size of capability request in bytes. */ - public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian - + public static final int CAPABILITY_REQUEST_SIZE = 8; /** - * Magic bytes for capability response: "ILP!" (ASCII). + * Size of capability response in bytes. */ - public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian - + public static final int CAPABILITY_RESPONSE_SIZE = 8; /** - * Magic bytes for fallback response (old server): "ILP0" (ASCII). + * Default initial receive buffer size (64 KB). */ - public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian - - // ==================== Header Structure ==================== - + public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024; /** - * Size of the message header in bytes. + * Default maximum batch size in bytes (16 MB). */ - public static final int HEADER_SIZE = 12; + public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024; /** - * Offset of magic bytes in header (4 bytes). + * Maximum in-flight batches for pipelining. */ - public static final int HEADER_OFFSET_MAGIC = 0; - + public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4; /** - * Offset of version byte in header. + * Default maximum rows per table in a batch. */ - public static final int HEADER_OFFSET_VERSION = 4; - + public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; /** - * Offset of flags byte in header. + * Default maximum string length in bytes (1 MB). */ - public static final int HEADER_OFFSET_FLAGS = 5; - + public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; /** - * Offset of table count (uint16, little-endian) in header. + * Default maximum tables per batch. */ - public static final int HEADER_OFFSET_TABLE_COUNT = 6; - + public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256; /** - * Offset of payload length (uint32, little-endian) in header. + * Flag bit: Delta symbol dictionary encoding enabled. + * When set, symbol columns use global IDs and send only new dictionary entries. */ - public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; - - // ==================== Protocol Version ==================== - + public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08; /** - * Current protocol version. + * Flag bit: Gorilla timestamp encoding enabled. */ - public static final byte VERSION_1 = 1; - - // ==================== Flag Bits ==================== + public static final byte FLAG_GORILLA = 0x04; /** * Flag bit: LZ4 compression enabled. @@ -101,295 +81,219 @@ public final class QwpConstants { * Flag bit: Zstd compression enabled. */ public static final byte FLAG_ZSTD = 0x02; - /** - * Flag bit: Gorilla timestamp encoding enabled. + * Mask for compression flags (bits 0-1). */ - public static final byte FLAG_GORILLA = 0x04; - + public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD; /** - * Flag bit: Delta symbol dictionary encoding enabled. - * When set, symbol columns use global IDs and send only new dictionary entries. + * Offset of flags byte in header. */ - public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08; - + public static final int HEADER_OFFSET_FLAGS = 5; /** - * Mask for compression flags (bits 0-1). + * Offset of magic bytes in header (4 bytes). */ - public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD; - - // ==================== Column Type Codes ==================== - + public static final int HEADER_OFFSET_MAGIC = 0; /** - * Column type: BOOLEAN (1 bit per value, packed). + * Offset of payload length (uint32, little-endian) in header. */ - public static final byte TYPE_BOOLEAN = 0x01; + public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; /** - * Column type: BYTE (int8). + * Offset of table count (uint16, little-endian) in header. */ - public static final byte TYPE_BYTE = 0x02; - + public static final int HEADER_OFFSET_TABLE_COUNT = 6; /** - * Column type: SHORT (int16, little-endian). + * Offset of version byte in header. */ - public static final byte TYPE_SHORT = 0x03; - + public static final int HEADER_OFFSET_VERSION = 4; /** - * Column type: INT (int32, little-endian). + * Size of the message header in bytes. */ - public static final byte TYPE_INT = 0x04; - + public static final int HEADER_SIZE = 12; /** - * Column type: LONG (int64, little-endian). + * Magic bytes for capability request: "ILP?" (ASCII). */ - public static final byte TYPE_LONG = 0x05; - + public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian /** - * Column type: FLOAT (IEEE 754 float32). + * Magic bytes for capability response: "ILP!" (ASCII). */ - public static final byte TYPE_FLOAT = 0x06; - + public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian /** - * Column type: DOUBLE (IEEE 754 float64). + * Magic bytes for fallback response (old server): "ILP0" (ASCII). */ - public static final byte TYPE_DOUBLE = 0x07; - + public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian /** - * Column type: STRING (length-prefixed UTF-8). + * Magic bytes for ILP v4 message: "ILP4" (ASCII). */ - public static final byte TYPE_STRING = 0x08; - + public static final int MAGIC_MESSAGE = 0x34504C49; // "ILP4" in little-endian /** - * Column type: SYMBOL (dictionary-encoded string). + * Maximum columns per table (QuestDB limit). */ - public static final byte TYPE_SYMBOL = 0x09; - + public static final int MAX_COLUMNS_PER_TABLE = 2048; /** - * Column type: TIMESTAMP (int64 microseconds since epoch). - * Use this for timestamps beyond nanosecond range (year > 2262). + * Maximum column name length in bytes. */ - public static final byte TYPE_TIMESTAMP = 0x0A; - + public static final int MAX_COLUMN_NAME_LENGTH = 127; /** - * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch). - * Use this for full nanosecond precision (limited to years 1677-2262). + * Maximum table name length in bytes. */ - public static final byte TYPE_TIMESTAMP_NANOS = 0x10; - + public static final int MAX_TABLE_NAME_LENGTH = 127; /** - * Column type: DATE (int64 milliseconds since epoch). + * Schema mode: Full schema included. */ - public static final byte TYPE_DATE = 0x0B; - + public static final byte SCHEMA_MODE_FULL = 0x00; /** - * Column type: UUID (16 bytes, big-endian). + * Schema mode: Schema reference (hash lookup). */ - public static final byte TYPE_UUID = 0x0C; - + public static final byte SCHEMA_MODE_REFERENCE = 0x01; /** - * Column type: LONG256 (32 bytes, big-endian). + * Status: Server error. */ - public static final byte TYPE_LONG256 = 0x0D; - + public static final byte STATUS_INTERNAL_ERROR = 0x06; /** - * Column type: GEOHASH (varint bits + packed geohash). + * Status: Batch accepted successfully. */ - public static final byte TYPE_GEOHASH = 0x0E; - + public static final byte STATUS_OK = 0x00; /** - * Column type: VARCHAR (length-prefixed UTF-8, aux storage). + * Status: Back-pressure, retry later. */ - public static final byte TYPE_VARCHAR = 0x0F; - + public static final byte STATUS_OVERLOADED = 0x07; /** - * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64). - * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + * Status: Malformed message. */ - public static final byte TYPE_DOUBLE_ARRAY = 0x11; - + public static final byte STATUS_PARSE_ERROR = 0x05; /** - * Column type: LONG_ARRAY (N-dimensional array of int64). - * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + * Status: Some rows failed (partial failure). */ - public static final byte TYPE_LONG_ARRAY = 0x12; - + public static final byte STATUS_PARTIAL = 0x01; /** - * Column type: DECIMAL64 (8 bytes, 18 digits precision). - * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)] + * Status: Column type incompatible. */ - public static final byte TYPE_DECIMAL64 = 0x13; - + public static final byte STATUS_SCHEMA_MISMATCH = 0x03; /** - * Column type: DECIMAL128 (16 bytes, 38 digits precision). - * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)] + * Status: Schema hash not recognized. */ - public static final byte TYPE_DECIMAL128 = 0x14; - + public static final byte STATUS_SCHEMA_REQUIRED = 0x02; /** - * Column type: DECIMAL256 (32 bytes, 77 digits precision). - * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)] + * Status: Table doesn't exist (auto-create disabled). */ - public static final byte TYPE_DECIMAL256 = 0x15; - + public static final byte STATUS_TABLE_NOT_FOUND = 0x04; /** - * Column type: CHAR (2-byte UTF-16 code unit). + * Column type: BOOLEAN (1 bit per value, packed). */ - public static final byte TYPE_CHAR = 0x16; - + public static final byte TYPE_BOOLEAN = 0x01; /** - * High bit indicating nullable column. + * Column type: BYTE (int8). */ - public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80; - + public static final byte TYPE_BYTE = 0x02; /** - * Mask for type code without nullable flag. + * Column type: CHAR (2-byte UTF-16 code unit). */ - public static final byte TYPE_MASK = 0x7F; - - // ==================== Schema Mode ==================== - + public static final byte TYPE_CHAR = 0x16; /** - * Schema mode: Full schema included. + * Column type: DATE (int64 milliseconds since epoch). */ - public static final byte SCHEMA_MODE_FULL = 0x00; + public static final byte TYPE_DATE = 0x0B; /** - * Schema mode: Schema reference (hash lookup). + * Column type: DECIMAL128 (16 bytes, 38 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)] */ - public static final byte SCHEMA_MODE_REFERENCE = 0x01; - - // ==================== Response Status Codes ==================== - + public static final byte TYPE_DECIMAL128 = 0x14; /** - * Status: Batch accepted successfully. + * Column type: DECIMAL256 (32 bytes, 77 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)] */ - public static final byte STATUS_OK = 0x00; + public static final byte TYPE_DECIMAL256 = 0x15; /** - * Status: Some rows failed (partial failure). + * Column type: DECIMAL64 (8 bytes, 18 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)] */ - public static final byte STATUS_PARTIAL = 0x01; - + public static final byte TYPE_DECIMAL64 = 0x13; /** - * Status: Schema hash not recognized. + * Column type: DOUBLE (IEEE 754 float64). */ - public static final byte STATUS_SCHEMA_REQUIRED = 0x02; - + public static final byte TYPE_DOUBLE = 0x07; /** - * Status: Column type incompatible. + * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] */ - public static final byte STATUS_SCHEMA_MISMATCH = 0x03; - + public static final byte TYPE_DOUBLE_ARRAY = 0x11; /** - * Status: Table doesn't exist (auto-create disabled). + * Column type: FLOAT (IEEE 754 float32). */ - public static final byte STATUS_TABLE_NOT_FOUND = 0x04; - + public static final byte TYPE_FLOAT = 0x06; /** - * Status: Malformed message. + * Column type: GEOHASH (varint bits + packed geohash). */ - public static final byte STATUS_PARSE_ERROR = 0x05; - + public static final byte TYPE_GEOHASH = 0x0E; /** - * Status: Server error. + * Column type: INT (int32, little-endian). */ - public static final byte STATUS_INTERNAL_ERROR = 0x06; - + public static final byte TYPE_INT = 0x04; /** - * Status: Back-pressure, retry later. + * Column type: LONG (int64, little-endian). */ - public static final byte STATUS_OVERLOADED = 0x07; - - // ==================== Default Limits ==================== - + public static final byte TYPE_LONG = 0x05; /** - * Default maximum batch size in bytes (16 MB). + * Column type: LONG256 (32 bytes, big-endian). */ - public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024; + public static final byte TYPE_LONG256 = 0x0D; /** - * Default maximum tables per batch. + * Column type: LONG_ARRAY (N-dimensional array of int64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] */ - public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256; - + public static final byte TYPE_LONG_ARRAY = 0x12; /** - * Default maximum rows per table in a batch. + * Mask for type code without nullable flag. */ - public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; - + public static final byte TYPE_MASK = 0x7F; /** - * Maximum columns per table (QuestDB limit). + * High bit indicating nullable column. */ - public static final int MAX_COLUMNS_PER_TABLE = 2048; - + public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80; /** - * Maximum table name length in bytes. + * Column type: SHORT (int16, little-endian). */ - public static final int MAX_TABLE_NAME_LENGTH = 127; - + public static final byte TYPE_SHORT = 0x03; /** - * Maximum column name length in bytes. + * Column type: STRING (length-prefixed UTF-8). */ - public static final int MAX_COLUMN_NAME_LENGTH = 127; - + public static final byte TYPE_STRING = 0x08; /** - * Default maximum string length in bytes (1 MB). + * Column type: SYMBOL (dictionary-encoded string). */ - public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; - + public static final byte TYPE_SYMBOL = 0x09; /** - * Default initial receive buffer size (64 KB). + * Column type: TIMESTAMP (int64 microseconds since epoch). + * Use this for timestamps beyond nanosecond range (year > 2262). */ - public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024; - + public static final byte TYPE_TIMESTAMP = 0x0A; /** - * Maximum in-flight batches for pipelining. + * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch). + * Use this for full nanosecond precision (limited to years 1677-2262). */ - public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4; - - // ==================== Capability Negotiation ==================== - + public static final byte TYPE_TIMESTAMP_NANOS = 0x10; /** - * Size of capability request in bytes. + * Column type: UUID (16 bytes, big-endian). */ - public static final int CAPABILITY_REQUEST_SIZE = 8; + public static final byte TYPE_UUID = 0x0C; /** - * Size of capability response in bytes. + * Column type: VARCHAR (length-prefixed UTF-8, aux storage). */ - public static final int CAPABILITY_RESPONSE_SIZE = 8; + public static final byte TYPE_VARCHAR = 0x0F; + /** + * Current protocol version. + */ + public static final byte VERSION_1 = 1; private QwpConstants() { // utility class } - /** - * Returns true if the type code represents a fixed-width type. - * - * @param typeCode the column type code (without nullable flag) - * @return true if fixed-width - */ - public static boolean isFixedWidthType(byte typeCode) { - int code = typeCode & TYPE_MASK; - return code == TYPE_BOOLEAN || - code == TYPE_BYTE || - code == TYPE_SHORT || - code == TYPE_CHAR || - code == TYPE_INT || - code == TYPE_LONG || - code == TYPE_FLOAT || - code == TYPE_DOUBLE || - code == TYPE_TIMESTAMP || - code == TYPE_TIMESTAMP_NANOS || - code == TYPE_DATE || - code == TYPE_UUID || - code == TYPE_LONG256 || - code == TYPE_DECIMAL64 || - code == TYPE_DECIMAL128 || - code == TYPE_DECIMAL256; - } - /** * Returns the size in bytes for fixed-width types. * @@ -509,4 +413,30 @@ public static String getTypeName(byte typeCode) { } return nullable ? name + "?" : name; } + + /** + * Returns true if the type code represents a fixed-width type. + * + * @param typeCode the column type code (without nullable flag) + * @return true if fixed-width + */ + public static boolean isFixedWidthType(byte typeCode) { + int code = typeCode & TYPE_MASK; + return code == TYPE_BOOLEAN || + code == TYPE_BYTE || + code == TYPE_SHORT || + code == TYPE_CHAR || + code == TYPE_INT || + code == TYPE_LONG || + code == TYPE_FLOAT || + code == TYPE_DOUBLE || + code == TYPE_TIMESTAMP || + code == TYPE_TIMESTAMP_NANOS || + code == TYPE_DATE || + code == TYPE_UUID || + code == TYPE_LONG256 || + code == TYPE_DECIMAL64 || + code == TYPE_DECIMAL128 || + code == TYPE_DECIMAL256; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 912af2d..082f6b2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -62,6 +62,84 @@ public class QwpGorillaEncoder { public QwpGorillaEncoder() { } + /** + * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap. + *

+ * Note: This does NOT include the encoding flag byte. Add 1 byte if + * the encoding flag is needed. + * + * @param srcAddress source address of contiguous int64 timestamps in native memory + * @param count number of timestamps + * @return encoded size in bytes (excluding encoding flag) + */ + public static int calculateEncodedSize(long srcAddress, int count) { + if (count == 0) { + return 0; + } + + int size = 8; // first timestamp + + if (count == 1) { + return size; + } + + size += 8; // second timestamp + + if (count == 2) { + return size; + } + + // Calculate bits for delta-of-delta encoding + long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8); + long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress); + int totalBits = 0; + + for (int i = 2; i < count; i++) { + long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8); + long delta = ts - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + totalBits += getBitsRequired(deltaOfDelta); + + prevDelta = delta; + prevTimestamp = ts; + } + + // Round up to bytes + size += (totalBits + 7) / 8; + + return size; + } + + /** + * Checks if Gorilla encoding can be used for timestamps stored off-heap. + *

+ * Gorilla encoding uses 32-bit signed integers for delta-of-delta values, + * so it cannot encode timestamps where the delta-of-delta exceeds the + * 32-bit signed integer range. + * + * @param srcAddress source address of contiguous int64 timestamps in native memory + * @param count number of timestamps + * @return true if Gorilla encoding can be used, false otherwise + */ + public static boolean canUseGorilla(long srcAddress, int count) { + if (count < 3) { + return true; // No DoD encoding needed for 0, 1, or 2 timestamps + } + + long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress); + for (int i = 2; i < count; i++) { + long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8) + - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8); + long dod = delta - prevDelta; + if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) { + return false; + } + prevDelta = delta; + } + return true; + } + /** * Returns the number of bits required to encode a delta-of-delta value. * @@ -209,82 +287,4 @@ public int encodeTimestamps(long destAddress, long capacity, long srcAddress, in return pos + bitWriter.finish(); } - - /** - * Checks if Gorilla encoding can be used for timestamps stored off-heap. - *

- * Gorilla encoding uses 32-bit signed integers for delta-of-delta values, - * so it cannot encode timestamps where the delta-of-delta exceeds the - * 32-bit signed integer range. - * - * @param srcAddress source address of contiguous int64 timestamps in native memory - * @param count number of timestamps - * @return true if Gorilla encoding can be used, false otherwise - */ - public static boolean canUseGorilla(long srcAddress, int count) { - if (count < 3) { - return true; // No DoD encoding needed for 0, 1, or 2 timestamps - } - - long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress); - for (int i = 2; i < count; i++) { - long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8) - - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8); - long dod = delta - prevDelta; - if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) { - return false; - } - prevDelta = delta; - } - return true; - } - - /** - * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap. - *

- * Note: This does NOT include the encoding flag byte. Add 1 byte if - * the encoding flag is needed. - * - * @param srcAddress source address of contiguous int64 timestamps in native memory - * @param count number of timestamps - * @return encoded size in bytes (excluding encoding flag) - */ - public static int calculateEncodedSize(long srcAddress, int count) { - if (count == 0) { - return 0; - } - - int size = 8; // first timestamp - - if (count == 1) { - return size; - } - - size += 8; // second timestamp - - if (count == 2) { - return size; - } - - // Calculate bits for delta-of-delta encoding - long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8); - long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress); - int totalBits = 0; - - for (int i = 2; i < count; i++) { - long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8); - long delta = ts - prevTimestamp; - long deltaOfDelta = delta - prevDelta; - - totalBits += getBitsRequired(deltaOfDelta); - - prevDelta = delta; - prevTimestamp = ts; - } - - // Round up to bytes - size += (totalBits + 7) / 8; - - return size; - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java index 90cf944..dd6020e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java @@ -49,70 +49,34 @@ private QwpNullBitmap() { } /** - * Calculates the size in bytes needed for a null bitmap. - * - * @param rowCount number of rows - * @return bitmap size in bytes - */ - public static int sizeInBytes(long rowCount) { - return (int) ((rowCount + 7) / 8); - } - - /** - * Checks if a specific row is null in the bitmap (from direct memory). + * Checks if all rows are null. * * @param address bitmap start address - * @param rowIndex row index to check - * @return true if the row is null + * @param rowCount total number of rows + * @return true if all rows are null */ - public static boolean isNull(long address, int rowIndex) { - int byteIndex = rowIndex >>> 3; // rowIndex / 8 - int bitIndex = rowIndex & 7; // rowIndex % 8 - byte b = Unsafe.getUnsafe().getByte(address + byteIndex); - return (b & (1 << bitIndex)) != 0; - } + public static boolean allNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; - /** - * Checks if a specific row is null in the bitmap (from byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowIndex row index to check - * @return true if the row is null - */ - public static boolean isNull(byte[] bitmap, int offset, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - byte b = bitmap[offset + byteIndex]; - return (b & (1 << bitIndex)) != 0; - } + // Check full bytes (all bits should be 1) + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + if ((b & 0xFF) != 0xFF) { + return false; + } + } - /** - * Sets a row as null in the bitmap (direct memory). - * - * @param address bitmap start address - * @param rowIndex row index to set as null - */ - public static void setNull(long address, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - long addr = address + byteIndex; - byte b = Unsafe.getUnsafe().getByte(addr); - b |= (1 << bitIndex); - Unsafe.getUnsafe().putByte(addr, b); - } + // Check remaining bits + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + if ((b & mask) != mask) { + return false; + } + } - /** - * Sets a row as null in the bitmap (byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowIndex row index to set as null - */ - public static void setNull(byte[] bitmap, int offset, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - bitmap[offset + byteIndex] |= (1 << bitIndex); + return true; } /** @@ -198,34 +162,81 @@ public static int countNulls(byte[] bitmap, int offset, int rowCount) { } /** - * Checks if all rows are null. + * Fills the bitmap setting all rows as null (direct memory). * * @param address bitmap start address * @param rowCount total number of rows - * @return true if all rows are null */ - public static boolean allNull(long address, int rowCount) { + public static void fillAllNull(long address, int rowCount) { int fullBytes = rowCount >>> 3; int remainingBits = rowCount & 7; - // Check full bytes (all bits should be 1) + // Fill full bytes with all 1s for (int i = 0; i < fullBytes; i++) { - byte b = Unsafe.getUnsafe().getByte(address + i); - if ((b & 0xFF) != 0xFF) { - return false; - } + Unsafe.getUnsafe().putByte(address + i, (byte) 0xFF); } - // Check remaining bits + // Set remaining bits in last byte if (remainingBits > 0) { - byte b = Unsafe.getUnsafe().getByte(address + fullBytes); - int mask = (1 << remainingBits) - 1; - if ((b & mask) != mask) { - return false; - } + byte mask = (byte) ((1 << remainingBits) - 1); + Unsafe.getUnsafe().putByte(address + fullBytes, mask); } + } - return true; + /** + * Clears the bitmap setting all rows as non-null (direct memory). + * + * @param address bitmap start address + * @param rowCount total number of rows + */ + public static void fillNoneNull(long address, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + Unsafe.getUnsafe().putByte(address + i, (byte) 0); + } + } + + /** + * Clears the bitmap setting all rows as non-null (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowCount total number of rows + */ + public static void fillNoneNull(byte[] bitmap, int offset, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + bitmap[offset + i] = 0; + } + } + + /** + * Checks if a specific row is null in the bitmap (from direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; // rowIndex / 8 + int bitIndex = rowIndex & 7; // rowIndex % 8 + byte b = Unsafe.getUnsafe().getByte(address + byteIndex); + return (b & (1 << bitIndex)) != 0; + } + + /** + * Checks if a specific row is null in the bitmap (from byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + byte b = bitmap[offset + byteIndex]; + return (b & (1 << bitIndex)) != 0; } /** @@ -260,51 +271,40 @@ public static boolean noneNull(long address, int rowCount) { } /** - * Fills the bitmap setting all rows as null (direct memory). + * Sets a row as null in the bitmap (direct memory). * * @param address bitmap start address - * @param rowCount total number of rows + * @param rowIndex row index to set as null */ - public static void fillAllNull(long address, int rowCount) { - int fullBytes = rowCount >>> 3; - int remainingBits = rowCount & 7; - - // Fill full bytes with all 1s - for (int i = 0; i < fullBytes; i++) { - Unsafe.getUnsafe().putByte(address + i, (byte) 0xFF); - } - - // Set remaining bits in last byte - if (remainingBits > 0) { - byte mask = (byte) ((1 << remainingBits) - 1); - Unsafe.getUnsafe().putByte(address + fullBytes, mask); - } + public static void setNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + long addr = address + byteIndex; + byte b = Unsafe.getUnsafe().getByte(addr); + b |= (1 << bitIndex); + Unsafe.getUnsafe().putByte(addr, b); } /** - * Clears the bitmap setting all rows as non-null (direct memory). + * Sets a row as null in the bitmap (byte array). * - * @param address bitmap start address - * @param rowCount total number of rows + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to set as null */ - public static void fillNoneNull(long address, int rowCount) { - int sizeBytes = sizeInBytes(rowCount); - for (int i = 0; i < sizeBytes; i++) { - Unsafe.getUnsafe().putByte(address + i, (byte) 0); - } + public static void setNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + bitmap[offset + byteIndex] |= (1 << bitIndex); } /** - * Clears the bitmap setting all rows as non-null (byte array). + * Calculates the size in bytes needed for a null bitmap. * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowCount total number of rows + * @param rowCount number of rows + * @return bitmap size in bytes */ - public static void fillNoneNull(byte[] bitmap, int offset, int rowCount) { - int sizeBytes = sizeInBytes(rowCount); - for (int i = 0; i < sizeBytes; i++) { - bitmap[offset + i] = 0; - } + public static int sizeInBytes(long rowCount) { + return (int) ((rowCount + 7) / 8); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 9edc3f6..dd6388a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -43,199 +43,21 @@ */ public final class QwpSchemaHash { + // Default seed (0 for ILP v4) + private static final long DEFAULT_SEED = 0L; // XXHash64 constants private static final long PRIME64_1 = 0x9E3779B185EBCA87L; private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL; + // Thread-local Hasher to avoid allocation on every computeSchemaHash call + private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); private static final long PRIME64_3 = 0x165667B19E3779F9L; private static final long PRIME64_4 = 0x85EBCA77C2B2AE63L; private static final long PRIME64_5 = 0x27D4EB2F165667C5L; - // Default seed (0 for ILP v4) - private static final long DEFAULT_SEED = 0L; - - // Thread-local Hasher to avoid allocation on every computeSchemaHash call - private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); - private QwpSchemaHash() { // utility class } - /** - * Computes XXHash64 of a byte array. - * - * @param data the data to hash - * @return the 64-bit hash value - */ - public static long hash(byte[] data) { - return hash(data, 0, data.length, DEFAULT_SEED); - } - - /** - * Computes XXHash64 of a byte array region. - * - * @param data the data to hash - * @param offset starting offset - * @param length number of bytes to hash - * @return the 64-bit hash value - */ - public static long hash(byte[] data, int offset, int length) { - return hash(data, offset, length, DEFAULT_SEED); - } - - /** - * Computes XXHash64 of a byte array region with custom seed. - * - * @param data the data to hash - * @param offset starting offset - * @param length number of bytes to hash - * @param seed the hash seed - * @return the 64-bit hash value - */ - public static long hash(byte[] data, int offset, int length, long seed) { - long h64; - int end = offset + length; - int pos = offset; - - if (length >= 32) { - int limit = end - 32; - long v1 = seed + PRIME64_1 + PRIME64_2; - long v2 = seed + PRIME64_2; - long v3 = seed; - long v4 = seed - PRIME64_1; - - do { - v1 = round(v1, getLong(data, pos)); - pos += 8; - v2 = round(v2, getLong(data, pos)); - pos += 8; - v3 = round(v3, getLong(data, pos)); - pos += 8; - v4 = round(v4, getLong(data, pos)); - pos += 8; - } while (pos <= limit); - - h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + - Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); - h64 = mergeRound(h64, v1); - h64 = mergeRound(h64, v2); - h64 = mergeRound(h64, v3); - h64 = mergeRound(h64, v4); - } else { - h64 = seed + PRIME64_5; - } - - h64 += length; - - // Process remaining 8-byte blocks - while (pos + 8 <= end) { - long k1 = getLong(data, pos); - k1 *= PRIME64_2; - k1 = Long.rotateLeft(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; - pos += 8; - } - - // Process remaining 4-byte block - if (pos + 4 <= end) { - h64 ^= (getInt(data, pos) & 0xFFFFFFFFL) * PRIME64_1; - h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; - pos += 4; - } - - // Process remaining bytes - while (pos < end) { - h64 ^= (data[pos] & 0xFFL) * PRIME64_5; - h64 = Long.rotateLeft(h64, 11) * PRIME64_1; - pos++; - } - - return avalanche(h64); - } - - /** - * Computes XXHash64 of direct memory. - * - * @param address start address - * @param length number of bytes - * @return the 64-bit hash value - */ - public static long hash(long address, long length) { - return hash(address, length, DEFAULT_SEED); - } - - /** - * Computes XXHash64 of direct memory with custom seed. - * - * @param address start address - * @param length number of bytes - * @param seed the hash seed - * @return the 64-bit hash value - */ - public static long hash(long address, long length, long seed) { - long h64; - long end = address + length; - long pos = address; - - if (length >= 32) { - long limit = end - 32; - long v1 = seed + PRIME64_1 + PRIME64_2; - long v2 = seed + PRIME64_2; - long v3 = seed; - long v4 = seed - PRIME64_1; - - do { - v1 = round(v1, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - v2 = round(v2, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - v3 = round(v3, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - v4 = round(v4, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - } while (pos <= limit); - - h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + - Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); - h64 = mergeRound(h64, v1); - h64 = mergeRound(h64, v2); - h64 = mergeRound(h64, v3); - h64 = mergeRound(h64, v4); - } else { - h64 = seed + PRIME64_5; - } - - h64 += length; - - // Process remaining 8-byte blocks - while (pos + 8 <= end) { - long k1 = Unsafe.getUnsafe().getLong(pos); - k1 *= PRIME64_2; - k1 = Long.rotateLeft(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; - pos += 8; - } - - // Process remaining 4-byte block - if (pos + 4 <= end) { - h64 ^= (Unsafe.getUnsafe().getInt(pos) & 0xFFFFFFFFL) * PRIME64_1; - h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; - pos += 4; - } - - // Process remaining bytes - while (pos < end) { - h64 ^= (Unsafe.getUnsafe().getByte(pos) & 0xFFL) * PRIME64_5; - h64 = Long.rotateLeft(h64, 11) * PRIME64_1; - pos++; - } - - return avalanche(h64); - } - /** * Computes the schema hash for ILP v4. *

@@ -377,18 +199,180 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList= 32) { + long limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v2 = round(v2, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v3 = round(v3, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v4 = round(v4, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = Unsafe.getUnsafe().getLong(pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (Unsafe.getUnsafe().getInt(pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (Unsafe.getUnsafe().getByte(pos) & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); } - private static long mergeRound(long acc, long val) { - val = round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; + /** + * Computes XXHash64 of a byte array. + * + * @param data the data to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data) { + return hash(data, 0, data.length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length) { + return hash(data, offset, length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region with custom seed. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length, long seed) { + long h64; + int end = offset + length; + int pos = offset; + + if (length >= 32) { + int limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, getLong(data, pos)); + pos += 8; + v2 = round(v2, getLong(data, pos)); + pos += 8; + v3 = round(v3, getLong(data, pos)); + pos += 8; + v4 = round(v4, getLong(data, pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = getLong(data, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (getInt(data, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (data[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes XXHash64 of direct memory. + * + * @param address start address + * @param length number of bytes + * @return the 64-bit hash value + */ + public static long hash(long address, long length) { + return hash(address, length, DEFAULT_SEED); } private static long avalanche(long h64) { @@ -400,6 +384,13 @@ private static long avalanche(long h64) { return h64; } + private static int getInt(byte[] data, int pos) { + return (data[pos] & 0xFF) | + ((data[pos + 1] & 0xFF) << 8) | + ((data[pos + 2] & 0xFF) << 16) | + ((data[pos + 3] & 0xFF) << 24); + } + private static long getLong(byte[] data, int pos) { return ((long) data[pos] & 0xFF) | (((long) data[pos + 1] & 0xFF) << 8) | @@ -411,11 +402,18 @@ private static long getLong(byte[] data, int pos) { (((long) data[pos + 7] & 0xFF) << 56); } - private static int getInt(byte[] data, int pos) { - return (data[pos] & 0xFF) | - ((data[pos + 1] & 0xFF) << 8) | - ((data[pos + 2] & 0xFF) << 16) | - ((data[pos + 3] & 0xFF) << 24); + private static long mergeRound(long acc, long val) { + val = round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; + } + + private static long round(long acc, long input) { + acc += input * PRIME64_2; + acc = Long.rotateLeft(acc, 31); + acc *= PRIME64_1; + return acc; } /** @@ -425,16 +423,64 @@ private static int getInt(byte[] data, int pos) { * as columns are processed. */ public static class Hasher { - private long v1, v2, v3, v4; - private long totalLen; private final byte[] buffer = new byte[32]; private int bufferPos; private long seed; + private long totalLen; + private long v1, v2, v3, v4; public Hasher() { reset(DEFAULT_SEED); } + /** + * Finalizes and returns the hash value. + * + * @return the 64-bit hash + */ + public long getValue() { + long h64; + + if (totalLen >= 32) { + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += totalLen; + + // Process buffered data + int pos = 0; + while (pos + 8 <= bufferPos) { + long k1 = getLong(buffer, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + if (pos + 4 <= bufferPos) { + h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + while (pos < bufferPos) { + h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + /** * Resets the hasher with the given seed. * @@ -450,20 +496,6 @@ public void reset(long seed) { bufferPos = 0; } - /** - * Updates the hash with a single byte. - * - * @param b the byte to add - */ - public void update(byte b) { - buffer[bufferPos++] = b; - totalLen++; - - if (bufferPos == 32) { - processBuffer(); - } - } - /** * Updates the hash with a byte array. * @@ -514,51 +546,17 @@ public void update(byte[] data, int offset, int length) { } /** - * Finalizes and returns the hash value. + * Updates the hash with a single byte. * - * @return the 64-bit hash + * @param b the byte to add */ - public long getValue() { - long h64; - - if (totalLen >= 32) { - h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + - Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); - h64 = mergeRound(h64, v1); - h64 = mergeRound(h64, v2); - h64 = mergeRound(h64, v3); - h64 = mergeRound(h64, v4); - } else { - h64 = seed + PRIME64_5; - } - - h64 += totalLen; - - // Process buffered data - int pos = 0; - while (pos + 8 <= bufferPos) { - long k1 = getLong(buffer, pos); - k1 *= PRIME64_2; - k1 = Long.rotateLeft(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; - pos += 8; - } - - if (pos + 4 <= bufferPos) { - h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1; - h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; - pos += 4; - } + public void update(byte b) { + buffer[bufferPos++] = b; + totalLen++; - while (pos < bufferPos) { - h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5; - h64 = Long.rotateLeft(h64, 11) * PRIME64_1; - pos++; + if (bufferPos == 32) { + processBuffer(); } - - return avalanche(h64); } private void processBuffer() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index b56cd28..4e47c1b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -294,23 +294,15 @@ static int elementSize(byte type) { * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr(). */ private static class ArrayCapture implements ArrayBufferAppender { + final int[] shape = new int[32]; double[] doubleData; int doubleDataOffset; - private boolean forLong; long[] longData; int longDataOffset; byte nDims; - final int[] shape = new int[32]; + private boolean forLong; private int shapeIndex; - void reset(boolean forLong) { - this.forLong = forLong; - shapeIndex = 0; - nDims = 0; - doubleDataOffset = 0; - longDataOffset = 0; - } - @Override public void putBlockOfBytes(long from, long len) { int count = (int) (len / 8); @@ -373,6 +365,14 @@ public void putLong(long value) { longData[longDataOffset++] = value; } } + + void reset(boolean forLong) { + this.forLong = forLong; + shapeIndex = 0; + nDims = 0; + doubleDataOffset = 0; + longDataOffset = 0; + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java index 3b98a70..ad675f4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java @@ -60,55 +60,6 @@ private QwpVarint() { // utility class } - /** - * Calculates the number of bytes needed to encode the given value. - * - * @param value the value to measure (treated as unsigned) - * @return number of bytes needed (1-10) - */ - public static int encodedLength(long value) { - if (value == 0) { - return 1; - } - // Count leading zeros to determine the number of bits needed - int bits = 64 - Long.numberOfLeadingZeros(value); - // Each byte encodes 7 bits, round up - return (bits + 6) / 7; - } - - /** - * Encodes a long value as a varint into the given byte array. - * - * @param buf the buffer to write to - * @param pos the position to start writing - * @param value the value to encode (treated as unsigned) - * @return the new position after the encoded bytes - */ - public static int encode(byte[] buf, int pos, long value) { - while ((value & ~DATA_MASK) != 0) { - buf[pos++] = (byte) ((value & DATA_MASK) | CONTINUATION_BIT); - value >>>= 7; - } - buf[pos++] = (byte) value; - return pos; - } - - /** - * Encodes a long value as a varint to direct memory. - * - * @param address the memory address to write to - * @param value the value to encode (treated as unsigned) - * @return the new address after the encoded bytes - */ - public static long encode(long address, long value) { - while ((value & ~DATA_MASK) != 0) { - Unsafe.getUnsafe().putByte(address++, (byte) ((value & DATA_MASK) | CONTINUATION_BIT)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(address++, (byte) value); - return address; - } - /** * Decodes a varint from the given byte array. * @@ -182,20 +133,6 @@ public static long decode(long address, long limit) { return result; } - /** - * Result holder for decoding varints when the number of bytes consumed matters. - * This class is mutable and should be reused to avoid allocations. - */ - public static class DecodeResult { - public long value; - public int bytesRead; - - public void reset() { - value = 0; - bytesRead = 0; - } - } - /** * Decodes a varint from a byte array and stores both value and bytes consumed. * @@ -258,4 +195,67 @@ public static void decode(long address, long limit, DecodeResult result) { result.value = value; result.bytesRead = bytesRead; } + + /** + * Encodes a long value as a varint to direct memory. + * + * @param address the memory address to write to + * @param value the value to encode (treated as unsigned) + * @return the new address after the encoded bytes + */ + public static long encode(long address, long value) { + while ((value & ~DATA_MASK) != 0) { + Unsafe.getUnsafe().putByte(address++, (byte) ((value & DATA_MASK) | CONTINUATION_BIT)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(address++, (byte) value); + return address; + } + + /** + * Encodes a long value as a varint into the given byte array. + * + * @param buf the buffer to write to + * @param pos the position to start writing + * @param value the value to encode (treated as unsigned) + * @return the new position after the encoded bytes + */ + public static int encode(byte[] buf, int pos, long value) { + while ((value & ~DATA_MASK) != 0) { + buf[pos++] = (byte) ((value & DATA_MASK) | CONTINUATION_BIT); + value >>>= 7; + } + buf[pos++] = (byte) value; + return pos; + } + + /** + * Calculates the number of bytes needed to encode the given value. + * + * @param value the value to measure (treated as unsigned) + * @return number of bytes needed (1-10) + */ + public static int encodedLength(long value) { + if (value == 0) { + return 1; + } + // Count leading zeros to determine the number of bits needed + int bits = 64 - Long.numberOfLeadingZeros(value); + // Each byte encodes 7 bits, round up + return (bits + 6) / 7; + } + + /** + * Result holder for decoding varints when the number of bytes consumed matters. + * This class is mutable and should be reused to avoid allocations. + */ + public static class DecodeResult { + public int bytesRead; + public long value; + + public void reset() { + value = 0; + bytesRead = 0; + } + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java index 44e596d..f113460 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java @@ -57,22 +57,22 @@ private QwpZigZag() { } /** - * Encodes a signed 64-bit integer using ZigZag encoding. + * Decodes a ZigZag encoded 64-bit integer. * - * @param value the signed value to encode - * @return the ZigZag encoded value (unsigned interpretation) + * @param value the ZigZag encoded value + * @return the original signed value */ - public static long encode(long value) { - return (value << 1) ^ (value >> 63); + public static long decode(long value) { + return (value >>> 1) ^ -(value & 1); } /** - * Decodes a ZigZag encoded 64-bit integer. + * Decodes a ZigZag encoded 32-bit integer. * * @param value the ZigZag encoded value * @return the original signed value */ - public static long decode(long value) { + public static int decode(int value) { return (value >>> 1) ^ -(value & 1); } @@ -87,12 +87,12 @@ public static int encode(int value) { } /** - * Decodes a ZigZag encoded 32-bit integer. + * Encodes a signed 64-bit integer using ZigZag encoding. * - * @param value the ZigZag encoded value - * @return the original signed value + * @param value the signed value to encode + * @return the ZigZag encoded value (unsigned interpretation) */ - public static int decode(int value) { - return (value >>> 1) ^ -(value & 1); + public static long encode(long value) { + return (value << 1) ^ (value >> 63); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 629767f..83253aa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -29,109 +29,77 @@ */ public final class WebSocketCloseCode { /** - * Normal closure (1000). - * The connection successfully completed whatever purpose for which it was created. + * Abnormal closure (1006). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that a connection was closed abnormally. */ - public static final int NORMAL_CLOSURE = 1000; - + public static final int ABNORMAL_CLOSURE = 1006; /** * Going away (1001). * The endpoint is going away, e.g., server shutting down or browser navigating away. */ public static final int GOING_AWAY = 1001; - /** - * Protocol error (1002). - * The endpoint is terminating the connection due to a protocol error. + * Internal server error (1011). + * The server encountered an unexpected condition that prevented it from fulfilling the request. */ - public static final int PROTOCOL_ERROR = 1002; - + public static final int INTERNAL_ERROR = 1011; /** - * Unsupported data (1003). - * The endpoint received a type of data it cannot accept. + * Invalid frame payload data (1007). + * The endpoint received a message with invalid payload data. */ - public static final int UNSUPPORTED_DATA = 1003; - + public static final int INVALID_PAYLOAD_DATA = 1007; /** - * Reserved (1004). - * Reserved for future use. + * Mandatory extension (1010). + * The client expected the server to negotiate one or more extensions. */ - public static final int RESERVED = 1004; - + public static final int MANDATORY_EXTENSION = 1010; /** - * No status received (1005). - * Reserved value. MUST NOT be sent in a Close frame. + * Message too big (1009). + * The endpoint received a message that is too big to process. */ - public static final int NO_STATUS_RECEIVED = 1005; - + public static final int MESSAGE_TOO_BIG = 1009; /** - * Abnormal closure (1006). - * Reserved value. MUST NOT be sent in a Close frame. - * Used to indicate that a connection was closed abnormally. + * Normal closure (1000). + * The connection successfully completed whatever purpose for which it was created. */ - public static final int ABNORMAL_CLOSURE = 1006; - + public static final int NORMAL_CLOSURE = 1000; /** - * Invalid frame payload data (1007). - * The endpoint received a message with invalid payload data. + * No status received (1005). + * Reserved value. MUST NOT be sent in a Close frame. */ - public static final int INVALID_PAYLOAD_DATA = 1007; - + public static final int NO_STATUS_RECEIVED = 1005; /** * Policy violation (1008). * The endpoint received a message that violates its policy. */ public static final int POLICY_VIOLATION = 1008; - - /** - * Message too big (1009). - * The endpoint received a message that is too big to process. - */ - public static final int MESSAGE_TOO_BIG = 1009; - /** - * Mandatory extension (1010). - * The client expected the server to negotiate one or more extensions. + * Protocol error (1002). + * The endpoint is terminating the connection due to a protocol error. */ - public static final int MANDATORY_EXTENSION = 1010; - + public static final int PROTOCOL_ERROR = 1002; /** - * Internal server error (1011). - * The server encountered an unexpected condition that prevented it from fulfilling the request. + * Reserved (1004). + * Reserved for future use. */ - public static final int INTERNAL_ERROR = 1011; - + public static final int RESERVED = 1004; /** * TLS handshake (1015). * Reserved value. MUST NOT be sent in a Close frame. * Used to indicate that the connection was closed due to TLS handshake failure. */ public static final int TLS_HANDSHAKE = 1015; + /** + * Unsupported data (1003). + * The endpoint received a type of data it cannot accept. + */ + public static final int UNSUPPORTED_DATA = 1003; private WebSocketCloseCode() { // Constants class } - /** - * Checks if a close code is valid for use in a Close frame. - * Codes 1005 and 1006 are reserved and must not be sent. - * - * @param code the close code - * @return true if the code can be sent in a Close frame - */ - public static boolean isValidForSending(int code) { - if (code < 1000) { - return false; - } - if (code == NO_STATUS_RECEIVED || code == ABNORMAL_CLOSURE || code == TLS_HANDSHAKE) { - return false; - } - // 1000-2999 are defined by RFC 6455 - // 3000-3999 are reserved for libraries/frameworks - // 4000-4999 are reserved for applications - return code < 5000; - } - /** * Returns a human-readable description of the close code. * @@ -175,4 +143,24 @@ public static String describe(int code) { return "Unknown (" + code + ")"; } } + + /** + * Checks if a close code is valid for use in a Close frame. + * Codes 1005 and 1006 are reserved and must not be sent. + * + * @param code the close code + * @return true if the code can be sent in a Close frame + */ + public static boolean isValidForSending(int code) { + if (code < 1000) { + return false; + } + if (code == NO_STATUS_RECEIVED || code == ABNORMAL_CLOSURE || code == TLS_HANDSHAKE) { + return false; + } + // 1000-2999 are defined by RFC 6455 + // 3000-3999 are reserved for libraries/frameworks + // 4000-4999 are reserved for applications + return code < 5000; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index fb980ec..e9d2d54 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -37,57 +37,82 @@ * have its own parser instance. */ public class WebSocketFrameParser { + /** + * Frame completely parsed. + */ + public static final int STATE_COMPLETE = 3; + /** + * Error state - frame is invalid. + */ + public static final int STATE_ERROR = 4; /** * Initial state, waiting for frame header. */ public static final int STATE_HEADER = 0; - /** * Need more data to complete parsing. */ public static final int STATE_NEED_MORE = 1; - /** * Header parsed, need payload data. */ public static final int STATE_NEED_PAYLOAD = 2; - - /** - * Frame completely parsed. - */ - public static final int STATE_COMPLETE = 3; - - /** - * Error state - frame is invalid. - */ - public static final int STATE_ERROR = 4; - // Frame header bits private static final int FIN_BIT = 0x80; - private static final int RSV_BITS = 0x70; - private static final int OPCODE_MASK = 0x0F; - private static final int MASK_BIT = 0x80; private static final int LENGTH_MASK = 0x7F; - + private static final int MASK_BIT = 0x80; // Control frame max payload size (RFC 6455) private static final int MAX_CONTROL_FRAME_PAYLOAD = 125; - + private static final int OPCODE_MASK = 0x0F; + private static final int RSV_BITS = 0x70; + private int errorCode; // Parsed frame data private boolean fin; - private int opcode; - private boolean masked; + private int headerSize; private int maskKey; + private boolean masked; + private int opcode; private long payloadLength; - private int headerSize; - - // Parser state - private int state = STATE_HEADER; - private int errorCode; - // Configuration private boolean serverMode = false; // If true, expect masked frames from clients + // Parser state + private int state = STATE_HEADER; private boolean strictMode = false; // If true, reject non-minimal length encodings + public int getErrorCode() { + return errorCode; + } + + public int getHeaderSize() { + return headerSize; + } + + public int getMaskKey() { + return maskKey; + } + + // Getters + + public int getOpcode() { + return opcode; + } + + public long getPayloadLength() { + return payloadLength; + } + + public int getState() { + return state; + } + + public boolean isFin() { + return fin; + } + + public boolean isMasked() { + return masked; + } + /** * Parses a WebSocket frame from the given buffer. * @@ -235,6 +260,38 @@ public int parse(long buf, long limit) { return (int) totalFrameSize; } + /** + * Resets the parser state for parsing a new frame. + */ + public void reset() { + state = STATE_HEADER; + fin = false; + opcode = 0; + masked = false; + maskKey = 0; + payloadLength = 0; + headerSize = 0; + errorCode = 0; + } + + /** + * Sets the mask key for unmasking. Used in testing. + */ + public void setMaskKey(int maskKey) { + this.maskKey = maskKey; + this.masked = true; + } + + // Setters for configuration + + public void setServerMode(boolean serverMode) { + this.serverMode = serverMode; + } + + public void setStrictMode(boolean strictMode) { + this.strictMode = strictMode; + } + /** * Unmasks the payload data in place. * @@ -273,70 +330,4 @@ public void unmaskPayload(long buf, long len) { i++; } } - - /** - * Resets the parser state for parsing a new frame. - */ - public void reset() { - state = STATE_HEADER; - fin = false; - opcode = 0; - masked = false; - maskKey = 0; - payloadLength = 0; - headerSize = 0; - errorCode = 0; - } - - // Getters - - public boolean isFin() { - return fin; - } - - public int getOpcode() { - return opcode; - } - - public boolean isMasked() { - return masked; - } - - public int getMaskKey() { - return maskKey; - } - - public long getPayloadLength() { - return payloadLength; - } - - public int getHeaderSize() { - return headerSize; - } - - public int getState() { - return state; - } - - public int getErrorCode() { - return errorCode; - } - - // Setters for configuration - - public void setServerMode(boolean serverMode) { - this.serverMode = serverMode; - } - - public void setStrictMode(boolean strictMode) { - this.strictMode = strictMode; - } - - /** - * Sets the mask key for unmasking. Used in testing. - */ - public void setMaskKey(int maskKey) { - this.maskKey = maskKey; - this.masked = true; - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index e4d423b..892fed4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -46,73 +46,105 @@ private WebSocketFrameWriter() { } /** - * Writes a WebSocket frame header to the buffer. + * Calculates the header size for a given payload length and masking. * - * @param buf the buffer to write to - * @param fin true if this is the final frame - * @param opcode the frame opcode * @param payloadLength the payload length - * @param masked true if the payload should be masked - * @return the number of bytes written (header size) + * @param masked true if the payload will be masked + * @return the header size in bytes */ - public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) { - int offset = 0; - - // First byte: FIN + opcode - int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F); - Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0); - - // Second byte: MASK + payload length - int maskBit = masked ? MASK_BIT : 0; - + public static int headerSize(long payloadLength, boolean masked) { + int size; if (payloadLength <= 125) { - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength)); + size = 2; } else if (payloadLength <= 65535) { - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126)); - Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF)); - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF)); + size = 4; } else { - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127)); - Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength)); - offset += 8; + size = 10; } + return masked ? size + 4 : size; + } - return offset; + /** + * Masks payload data in place using XOR with the given mask key. + * + * @param buf the payload buffer + * @param len the payload length + * @param maskKey the 4-byte mask key + */ + public static void maskPayload(long buf, long len, int maskKey) { + // Process 8 bytes at a time when possible + long i = 0; + long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); + + // Process 8-byte chunks + while (i + 8 <= len) { + long value = Unsafe.getUnsafe().getLong(buf + i); + Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); + i += 8; + } + + // Process 4-byte chunk if remaining + if (i + 4 <= len) { + int value = Unsafe.getUnsafe().getInt(buf + i); + Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); + i += 4; + } + + // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation + while (i < len) { + byte b = Unsafe.getUnsafe().getByte(buf + i); + int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF; + Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); + i++; + } } /** - * Writes a WebSocket frame header with optional mask key. + * Writes a binary frame with payload from a memory address. * - * @param buf the buffer to write to - * @param fin true if this is the final frame - * @param opcode the frame opcode - * @param payloadLength the payload length - * @param maskKey the mask key (only used if masked is true) - * @return the number of bytes written (header size including mask key) + * @param buf the buffer to write to + * @param payloadPtr pointer to the payload data + * @param payloadLen length of payload + * @return the total number of bytes written */ - public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { - int offset = writeHeader(buf, fin, opcode, payloadLength, true); - Unsafe.getUnsafe().putInt(buf + offset, maskKey); - return offset + 4; + public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + + // Copy payload from memory + Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); + + return headerLen + payloadLen; } /** - * Calculates the header size for a given payload length and masking. + * Writes a binary frame header only (for when payload is written separately). * - * @param payloadLength the payload length - * @param masked true if the payload will be masked + * @param buf the buffer to write to + * @param payloadLen length of payload that will follow * @return the header size in bytes */ - public static int headerSize(long payloadLength, boolean masked) { - int size; - if (payloadLength <= 125) { - size = 2; - } else if (payloadLength <= 65535) { - size = 4; - } else { - size = 10; + public static int writeBinaryFrameHeader(long buf, int payloadLen) { + return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + } + + /** + * Writes a complete Close frame to the buffer. + * + * @param buf the buffer to write to + * @param code the close status code + * @param reason the close reason (may be null) + * @return the total number of bytes written (header + payload) + */ + public static int writeCloseFrame(long buf, int code, String reason) { + int payloadLen = 2; // status code + if (reason != null && !reason.isEmpty()) { + payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; } - return masked ? size + 4 : size; + + int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); + int payloadOffset = writeClosePayload(buf + headerLen, code, reason); + + return headerLen + payloadOffset; } /** @@ -140,32 +172,63 @@ public static int writeClosePayload(long buf, int code, String reason) { } /** - * Writes a complete Close frame to the buffer. + * Writes a WebSocket frame header to the buffer. * - * @param buf the buffer to write to - * @param code the close status code - * @param reason the close reason (may be null) - * @return the total number of bytes written (header + payload) + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param masked true if the payload should be masked + * @return the number of bytes written (header size) */ - public static int writeCloseFrame(long buf, int code, String reason) { - int payloadLen = 2; // status code - if (reason != null && !reason.isEmpty()) { - payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) { + int offset = 0; + + // First byte: FIN + opcode + int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0); + + // Second byte: MASK + payload length + int maskBit = masked ? MASK_BIT : 0; + + if (payloadLength <= 125) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength)); + } else if (payloadLength <= 65535) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF)); + } else { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127)); + Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength)); + offset += 8; } - int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); - int payloadOffset = writeClosePayload(buf + headerLen, code, reason); + return offset; + } - return headerLen + payloadOffset; + /** + * Writes a WebSocket frame header with optional mask key. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param maskKey the mask key (only used if masked is true) + * @return the number of bytes written (header size including mask key) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { + int offset = writeHeader(buf, fin, opcode, payloadLength, true); + Unsafe.getUnsafe().putInt(buf + offset, maskKey); + return offset + 4; } /** * Writes a complete Ping frame to the buffer. * - * @param buf the buffer to write to - * @param payload the ping payload - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write + * @param buf the buffer to write to + * @param payload the ping payload + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write * @return the total number of bytes written */ public static int writePingFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { @@ -182,10 +245,10 @@ public static int writePingFrame(long buf, byte[] payload, int payloadOff, int p /** * Writes a complete Pong frame to the buffer. * - * @param buf the buffer to write to - * @param payload the pong payload (should match the received ping) - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write + * @param buf the buffer to write to + * @param payload the pong payload (should match the received ping) + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write * @return the total number of bytes written */ public static int writePongFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { @@ -215,67 +278,4 @@ public static int writePongFrame(long buf, long payloadPtr, int payloadLen) { return headerLen + payloadLen; } - - /** - * Writes a binary frame with payload from a memory address. - * - * @param buf the buffer to write to - * @param payloadPtr pointer to the payload data - * @param payloadLen length of payload - * @return the total number of bytes written - */ - public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - - // Copy payload from memory - Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); - - return headerLen + payloadLen; - } - - /** - * Writes a binary frame header only (for when payload is written separately). - * - * @param buf the buffer to write to - * @param payloadLen length of payload that will follow - * @return the header size in bytes - */ - public static int writeBinaryFrameHeader(long buf, int payloadLen) { - return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - } - - /** - * Masks payload data in place using XOR with the given mask key. - * - * @param buf the payload buffer - * @param len the payload length - * @param maskKey the 4-byte mask key - */ - public static void maskPayload(long buf, long len, int maskKey) { - // Process 8 bytes at a time when possible - long i = 0; - long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); - - // Process 8-byte chunks - while (i + 8 <= len) { - long value = Unsafe.getUnsafe().getLong(buf + i); - Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); - i += 8; - } - - // Process 4-byte chunk if remaining - if (i + 4 <= len) { - int value = Unsafe.getUnsafe().getInt(buf + i); - Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); - i += 4; - } - - // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation - while (i < len) { - byte b = Unsafe.getUnsafe().getByte(buf + i); - int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF; - Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); - i++; - } - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java index 0bbcddd..e87cb1d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java @@ -40,28 +40,24 @@ * generating proper handshake responses. */ public final class WebSocketHandshake { + public static final Utf8String HEADER_CONNECTION = new Utf8String("Connection"); + public static final Utf8String HEADER_SEC_WEBSOCKET_ACCEPT = new Utf8String("Sec-WebSocket-Accept"); + public static final Utf8String HEADER_SEC_WEBSOCKET_KEY = new Utf8String("Sec-WebSocket-Key"); + public static final Utf8String HEADER_SEC_WEBSOCKET_PROTOCOL = new Utf8String("Sec-WebSocket-Protocol"); + public static final Utf8String HEADER_SEC_WEBSOCKET_VERSION = new Utf8String("Sec-WebSocket-Version"); + // Header names (case-insensitive) + public static final Utf8String HEADER_UPGRADE = new Utf8String("Upgrade"); + public static final Utf8String VALUE_UPGRADE = new Utf8String("upgrade"); + // Header values + public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); /** * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. */ public static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; - /** * The required WebSocket version (RFC 6455). */ public static final int WEBSOCKET_VERSION = 13; - - // Header names (case-insensitive) - public static final Utf8String HEADER_UPGRADE = new Utf8String("Upgrade"); - public static final Utf8String HEADER_CONNECTION = new Utf8String("Connection"); - public static final Utf8String HEADER_SEC_WEBSOCKET_KEY = new Utf8String("Sec-WebSocket-Key"); - public static final Utf8String HEADER_SEC_WEBSOCKET_VERSION = new Utf8String("Sec-WebSocket-Version"); - public static final Utf8String HEADER_SEC_WEBSOCKET_PROTOCOL = new Utf8String("Sec-WebSocket-Protocol"); - public static final Utf8String HEADER_SEC_WEBSOCKET_ACCEPT = new Utf8String("Sec-WebSocket-Accept"); - - // Header values - public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); - public static final Utf8String VALUE_UPGRADE = new Utf8String("upgrade"); - // Response template private static final byte[] RESPONSE_PREFIX = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ".getBytes(StandardCharsets.US_ASCII); @@ -81,13 +77,45 @@ private WebSocketHandshake() { } /** - * Checks if the given header indicates a WebSocket upgrade request. + * Computes the Sec-WebSocket-Accept value for the given key. * - * @param upgradeHeader the value of the Upgrade header - * @return true if this is a WebSocket upgrade request + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response */ - public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { - return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); + public static String computeAcceptKey(Utf8Sequence key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + byte[] keyBytes = new byte[key.size()]; + for (int i = 0; i < key.size(); i++) { + keyBytes[i] = key.byteAt(i); + } + sha1.update(keyBytes); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); + } + + /** + * Computes the Sec-WebSocket-Accept value for the given key string. + * + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response + */ + public static String computeAcceptKey(String key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + sha1.update(key.getBytes(StandardCharsets.US_ASCII)); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); } /** @@ -106,38 +134,32 @@ public static boolean isConnectionUpgrade(Utf8Sequence connectionHeader) { } /** - * Checks if the sequence contains the given substring (case-insensitive). + * Validates the Sec-WebSocket-Key header. + * The key must be a base64-encoded 16-byte value. + * + * @param key the Sec-WebSocket-Key header value + * @return true if the key is valid */ - private static boolean containsIgnoreCaseAscii(Utf8Sequence seq, Utf8Sequence substring) { - int seqLen = seq.size(); - int subLen = substring.size(); - - if (subLen > seqLen) { + public static boolean isValidKey(Utf8Sequence key) { + if (key == null) { return false; } - if (subLen == 0) { - return true; + // Base64-encoded 16-byte value should be exactly 24 characters + // (16 bytes = 128 bits = 22 base64 chars + 2 padding = 24) + int size = key.size(); + if (size != 24) { + return false; } - - outer: - for (int i = 0; i <= seqLen - subLen; i++) { - for (int j = 0; j < subLen; j++) { - byte a = seq.byteAt(i + j); - byte b = substring.byteAt(j); - // Convert to lowercase for comparison - if (a >= 'A' && a <= 'Z') { - a = (byte) (a + 32); - } - if (b >= 'A' && b <= 'Z') { - b = (byte) (b + 32); - } - if (a != b) { - continue outer; - } + // Basic validation: check that all characters are valid base64 + for (int i = 0; i < size; i++) { + byte b = key.byteAt(i); + boolean valid = (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || + (b >= '0' && b <= '9') || b == '+' || b == '/' || b == '='; + if (!valid) { + return false; } - return true; } - return false; + return true; } /** @@ -167,74 +189,101 @@ public static boolean isValidVersion(Utf8Sequence versionHeader) { } /** - * Validates the Sec-WebSocket-Key header. - * The key must be a base64-encoded 16-byte value. + * Checks if the given header indicates a WebSocket upgrade request. * - * @param key the Sec-WebSocket-Key header value - * @return true if the key is valid + * @param upgradeHeader the value of the Upgrade header + * @return true if this is a WebSocket upgrade request */ - public static boolean isValidKey(Utf8Sequence key) { - if (key == null) { - return false; - } - // Base64-encoded 16-byte value should be exactly 24 characters - // (16 bytes = 128 bits = 22 base64 chars + 2 padding = 24) - int size = key.size(); - if (size != 24) { - return false; - } - // Basic validation: check that all characters are valid base64 - for (int i = 0; i < size; i++) { - byte b = key.byteAt(i); - boolean valid = (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || - (b >= '0' && b <= '9') || b == '+' || b == '/' || b == '='; - if (!valid) { - return false; - } - } - return true; + public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { + return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); } /** - * Computes the Sec-WebSocket-Accept value for the given key. + * Returns the size of the handshake response for the given accept key. * - * @param key the Sec-WebSocket-Key from the client - * @return the base64-encoded SHA-1 hash to send in the response + * @param acceptKey the computed accept key + * @return the total response size in bytes */ - public static String computeAcceptKey(Utf8Sequence key) { - MessageDigest sha1 = SHA1_DIGEST.get(); - sha1.reset(); + public static int responseSize(String acceptKey) { + return RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + } - // Concatenate key + GUID - byte[] keyBytes = new byte[key.size()]; - for (int i = 0; i < key.size(); i++) { - keyBytes[i] = key.byteAt(i); + /** + * Returns the size of the handshake response with an optional subprotocol. + * + * @param acceptKey the computed accept key + * @param protocol the negotiated subprotocol (may be null or empty) + * @return the total response size in bytes + */ + public static int responseSizeWithProtocol(String acceptKey, String protocol) { + int size = RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + if (protocol != null && !protocol.isEmpty()) { + size += "\r\nSec-WebSocket-Protocol: ".length() + protocol.length(); } - sha1.update(keyBytes); - sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + return size; + } - // Compute SHA-1 hash and base64 encode - byte[] hash = sha1.digest(); - return Base64.getEncoder().encodeToString(hash); + /** + * Validates all required headers for a WebSocket upgrade request. + * + * @param upgradeHeader the Upgrade header value + * @param connectionHeader the Connection header value + * @param keyHeader the Sec-WebSocket-Key header value + * @param versionHeader the Sec-WebSocket-Version header value + * @return null if valid, or an error message describing the problem + */ + public static String validate( + Utf8Sequence upgradeHeader, + Utf8Sequence connectionHeader, + Utf8Sequence keyHeader, + Utf8Sequence versionHeader + ) { + if (!isWebSocketUpgrade(upgradeHeader)) { + return "Missing or invalid Upgrade header"; + } + if (!isConnectionUpgrade(connectionHeader)) { + return "Missing or invalid Connection header"; + } + if (!isValidKey(keyHeader)) { + return "Missing or invalid Sec-WebSocket-Key header"; + } + if (!isValidVersion(versionHeader)) { + return "Unsupported WebSocket version"; + } + return null; } /** - * Computes the Sec-WebSocket-Accept value for the given key string. + * Writes a 400 Bad Request response. * - * @param key the Sec-WebSocket-Key from the client - * @return the base64-encoded SHA-1 hash to send in the response + * @param buf the buffer to write to + * @param reason the reason for the bad request + * @return the number of bytes written */ - public static String computeAcceptKey(String key) { - MessageDigest sha1 = SHA1_DIGEST.get(); - sha1.reset(); + public static int writeBadRequestResponse(long buf, String reason) { + int offset = 0; - // Concatenate key + GUID - sha1.update(key.getBytes(StandardCharsets.US_ASCII)); - sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + byte[] statusLine = "HTTP/1.1 400 Bad Request\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : statusLine) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } - // Compute SHA-1 hash and base64 encode - byte[] hash = sha1.digest(); - return Base64.getEncoder().encodeToString(hash); + byte[] contentType = "Content-Type: text/plain\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : contentType) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] reasonBytes = reason != null ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] contentLength = ("Content-Length: " + reasonBytes.length + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII); + for (byte b : contentLength) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + for (byte b : reasonBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; } /** @@ -266,16 +315,6 @@ public static int writeResponse(long buf, String acceptKey) { return offset; } - /** - * Returns the size of the handshake response for the given accept key. - * - * @param acceptKey the computed accept key - * @return the total response size in bytes - */ - public static int responseSize(String acceptKey) { - return RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; - } - /** * Writes the WebSocket handshake response with an optional subprotocol. * @@ -314,54 +353,6 @@ public static int writeResponseWithProtocol(long buf, String acceptKey, String p return offset; } - /** - * Returns the size of the handshake response with an optional subprotocol. - * - * @param acceptKey the computed accept key - * @param protocol the negotiated subprotocol (may be null or empty) - * @return the total response size in bytes - */ - public static int responseSizeWithProtocol(String acceptKey, String protocol) { - int size = RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; - if (protocol != null && !protocol.isEmpty()) { - size += "\r\nSec-WebSocket-Protocol: ".length() + protocol.length(); - } - return size; - } - - /** - * Writes a 400 Bad Request response. - * - * @param buf the buffer to write to - * @param reason the reason for the bad request - * @return the number of bytes written - */ - public static int writeBadRequestResponse(long buf, String reason) { - int offset = 0; - - byte[] statusLine = "HTTP/1.1 400 Bad Request\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : statusLine) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] contentType = "Content-Type: text/plain\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : contentType) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] reasonBytes = reason != null ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; - byte[] contentLength = ("Content-Length: " + reasonBytes.length + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII); - for (byte b : contentLength) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - for (byte b : reasonBytes) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - return offset; - } - /** * Writes a 426 Upgrade Required response indicating unsupported WebSocket version. * @@ -390,32 +381,37 @@ public static int writeVersionNotSupportedResponse(long buf) { } /** - * Validates all required headers for a WebSocket upgrade request. - * - * @param upgradeHeader the Upgrade header value - * @param connectionHeader the Connection header value - * @param keyHeader the Sec-WebSocket-Key header value - * @param versionHeader the Sec-WebSocket-Version header value - * @return null if valid, or an error message describing the problem + * Checks if the sequence contains the given substring (case-insensitive). */ - public static String validate( - Utf8Sequence upgradeHeader, - Utf8Sequence connectionHeader, - Utf8Sequence keyHeader, - Utf8Sequence versionHeader - ) { - if (!isWebSocketUpgrade(upgradeHeader)) { - return "Missing or invalid Upgrade header"; - } - if (!isConnectionUpgrade(connectionHeader)) { - return "Missing or invalid Connection header"; + private static boolean containsIgnoreCaseAscii(Utf8Sequence seq, Utf8Sequence substring) { + int seqLen = seq.size(); + int subLen = substring.size(); + + if (subLen > seqLen) { + return false; } - if (!isValidKey(keyHeader)) { - return "Missing or invalid Sec-WebSocket-Key header"; + if (subLen == 0) { + return true; } - if (!isValidVersion(versionHeader)) { - return "Unsupported WebSocket version"; + + outer: + for (int i = 0; i <= seqLen - subLen; i++) { + for (int j = 0; j < subLen; j++) { + byte a = seq.byteAt(i + j); + byte b = substring.byteAt(j); + // Convert to lowercase for comparison + if (a >= 'A' && a <= 'Z') { + a = (byte) (a + 32); + } + if (b >= 'A' && b <= 'Z') { + b = (byte) (b + 32); + } + if (a != b) { + continue outer; + } + } + return true; } - return null; + return false; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java index 40466ec..f2fead7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -28,43 +28,38 @@ * WebSocket frame opcodes as defined in RFC 6455. */ public final class WebSocketOpcode { - /** - * Continuation frame (0x0). - * Used for fragmented messages after the initial frame. - */ - public static final int CONTINUATION = 0x00; - - /** - * Text frame (0x1). - * Payload is UTF-8 encoded text. - */ - public static final int TEXT = 0x01; - /** * Binary frame (0x2). * Payload is arbitrary binary data. */ public static final int BINARY = 0x02; - - // Reserved non-control frames: 0x3-0x7 - /** * Connection close frame (0x8). * Indicates that the endpoint wants to close the connection. */ public static final int CLOSE = 0x08; + /** + * Continuation frame (0x0). + * Used for fragmented messages after the initial frame. + */ + public static final int CONTINUATION = 0x00; + // Reserved non-control frames: 0x3-0x7 /** * Ping frame (0x9). * Used for keep-alive and connection health checks. */ public static final int PING = 0x09; - /** * Pong frame (0xA). * Response to a ping frame. */ public static final int PONG = 0x0A; + /** + * Text frame (0x1). + * Payload is UTF-8 encoded text. + */ + public static final int TEXT = 0x01; // Reserved control frames: 0xB-0xF diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index 1f35299..a3f7939 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -106,18 +106,6 @@ public static long getAddrInfo(CharSequence host, int port) { } } - private static long getAddrInfo(DirectUtf8Sequence host, int port) { - return getAddrInfo(host.ptr(), port); - } - - private static long getAddrInfo(long lpszHost, int port) { - long addrInfo = getAddrInfo0(lpszHost, port); - if (addrInfo != -1) { - ADDR_INFO_COUNTER.incrementAndGet(); - } - return addrInfo; - } - public native static int getSndBuf(int fd); public static void init() { @@ -163,6 +151,18 @@ public static long sockaddr(int ipv4address, int port) { private static native void freeSockAddr0(long sockaddr); + private static long getAddrInfo(DirectUtf8Sequence host, int port) { + return getAddrInfo(host.ptr(), port); + } + + private static long getAddrInfo(long lpszHost, int port) { + long addrInfo = getAddrInfo0(lpszHost, port); + if (addrInfo != -1) { + ADDR_INFO_COUNTER.incrementAndGet(); + } + return addrInfo; + } + private static native long getAddrInfo0(long lpszHost, int port); private static native int getEwouldblock(); @@ -186,4 +186,4 @@ public static long sockaddr(int ipv4address, int port) { MMSGHDR_BUFFER_LENGTH_OFFSET = -1L; } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java index d299423..932d1eb 100644 --- a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -157,6 +157,29 @@ public int valueQuick(int index) { return get(list.getQuick(index)); } + private void erase(int index) { + keys[index] = noEntryKey; + values[index] = noEntryValue; + } + + private void move(int from, int to) { + keys[to] = keys[from]; + values[to] = values[from]; + erase(from); + } + + private int probe0(CharSequence key, int index) { + do { + index = (index + 1) & mask; + if (keys[index] == noEntryKey) { + return index; + } + if (Chars.equals(key, keys[index])) { + return -index - 1; + } + } while (true); + } + private void putAt0(int index, CharSequence key, int value) { keys[index] = key; values[index] = value; @@ -183,27 +206,4 @@ private void rehash() { } } } - - private void erase(int index) { - keys[index] = noEntryKey; - values[index] = noEntryValue; - } - - private void move(int from, int to) { - keys[to] = keys[from]; - values[to] = values[from]; - erase(from); - } - - private int probe0(CharSequence key, int index) { - do { - index = (index + 1) & mask; - if (keys[index] == noEntryKey) { - return index; - } - if (Chars.equals(key, keys[index])) { - return -index - 1; - } - } while (true); - } } diff --git a/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java b/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java index e9f1f23..3d8b130 100644 --- a/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java +++ b/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java @@ -71,6 +71,7 @@ public long ptr() { return impl; } }; + public DirectByteSink(long initialCapacity, int memoryTag) { this(initialCapacity, memoryTag, false); } @@ -275,4 +276,4 @@ private void setImplPtr(long ptr) { static { Os.init(); } -} \ No newline at end of file +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 5eab4bf..88715be 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -65,34 +65,30 @@ */ public class QwpAllocationTestClient { - // Protocol modes - private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; - private static final String PROTOCOL_ILP_HTTP = "ilp-http"; - private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; - - - // Default configuration - private static final String DEFAULT_HOST = "localhost"; - private static final int DEFAULT_ROWS = 80_000_000; private static final int DEFAULT_BATCH_SIZE = 10_000; private static final int DEFAULT_FLUSH_BYTES = 0; // 0 = use protocol default private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; // 0 = use protocol default + // Default configuration + private static final String DEFAULT_HOST = "localhost"; private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8) + private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; + private static final int DEFAULT_ROWS = 80_000_000; private static final int DEFAULT_SEND_QUEUE = 0; // 0 = use protocol default (16) private static final int DEFAULT_WARMUP_ROWS = 100_000; - private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; - - // Pre-computed test data to avoid allocation during the test - private static final String[] SYMBOLS = { - "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ", - "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA" - }; - + private static final String PROTOCOL_ILP_HTTP = "ilp-http"; + // Protocol modes + private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; + private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; private static final String[] STRINGS = { "New York", "London", "Tokyo", "Paris", "Berlin", "Sydney", "Toronto", "Singapore", "Hong Kong", "Dubai", "Mumbai", "Shanghai", "Moscow", "Seoul", "Bangkok", "Amsterdam", "Zurich", "Frankfurt", "Milan", "Madrid" }; + // Pre-computed test data to avoid allocation during the test + private static final String[] SYMBOLS = { + "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ", + "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA" + }; public static void main(String[] args) { // Parse command-line options @@ -176,6 +172,61 @@ public static void main(String[] args) { } } + private static Sender createSender(String protocol, String host, int port, + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, int sendQueue) { + switch (protocol) { + case PROTOCOL_ILP_TCP: + return Sender.builder(Sender.Transport.TCP) + .address(host) + .port(port) + .build(); + case PROTOCOL_ILP_HTTP: + return Sender.builder(Sender.Transport.HTTP) + .address(host) + .port(port) + .autoFlushRows(batchSize) + .build(); + case PROTOCOL_QWP_WEBSOCKET: + Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) + .address(host) + .port(port) + .asyncMode(true); + if (batchSize > 0) b.autoFlushRows(batchSize); + if (flushBytes > 0) b.autoFlushBytes(flushBytes); + if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); + if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); + if (sendQueue > 0) b.sendQueueCapacity(sendQueue); + return b.build(); + default: + throw new IllegalArgumentException("Unknown protocol: " + protocol + + ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); + } + } + + /** + * Estimates the size of a single row in bytes for throughput calculation. + */ + private static int estimatedRowSize() { + // Rough estimate (binary protocol): + // - 2 symbols: ~10 bytes each = 20 bytes + // - 3 longs: 8 bytes each = 24 bytes + // - 4 doubles: 8 bytes each = 32 bytes + // - 1 string: ~10 bytes average + // - 1 boolean: 1 byte + // - 2 timestamps: 8 bytes each = 16 bytes + // - Overhead: ~20 bytes + // Total: ~123 bytes + return 123; + } + + private static int getDefaultPort(String protocol) { + if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { + return 9000; + } + return 9009; + } + private static void printUsage() { System.out.println("ILP Allocation Test Client"); System.out.println(); @@ -207,17 +258,10 @@ private static void printUsage() { System.out.println(" QwpAllocationTestClient --protocol=ilp-tcp --rows=100000 --no-warmup"); } - private static int getDefaultPort(String protocol) { - if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { - return 9000; - } - return 9009; - } - private static void runTest(String protocol, String host, int port, int totalRows, - int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue, - int warmupRows, int reportInterval) throws IOException { + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, int sendQueue, + int warmupRows, int reportInterval) throws IOException { System.out.println("Connecting to " + host + ":" + port + "..."); try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, @@ -289,7 +333,7 @@ private static void runTest(String protocol, String host, int port, int totalRow System.out.println("Batch size: " + String.format("%,d", batchSize)); System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds"); System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second"); - System.out.println("Data rate (before compression): " + String.format("%.2f", ((long)totalRows * estimatedRowSize()) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)"); + System.out.println("Data rate (before compression): " + String.format("%.2f", ((long) totalRows * estimatedRowSize()) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -297,38 +341,6 @@ private static void runTest(String protocol, String host, int port, int totalRow } } - private static Sender createSender(String protocol, String host, int port, - int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { - switch (protocol) { - case PROTOCOL_ILP_TCP: - return Sender.builder(Sender.Transport.TCP) - .address(host) - .port(port) - .build(); - case PROTOCOL_ILP_HTTP: - return Sender.builder(Sender.Transport.HTTP) - .address(host) - .port(port) - .autoFlushRows(batchSize) - .build(); - case PROTOCOL_QWP_WEBSOCKET: - Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) - .address(host) - .port(port) - .asyncMode(true); - if (batchSize > 0) b.autoFlushRows(batchSize); - if (flushBytes > 0) b.autoFlushBytes(flushBytes); - if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); - if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); - return b.build(); - default: - throw new IllegalArgumentException("Unknown protocol: " + protocol + - ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); - } - } - private static void sendRow(Sender sender, int rowIndex) { // Base timestamp with small variations long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros @@ -360,20 +372,4 @@ private static void sendRow(Sender sender, int rowIndex) { // Designated timestamp .at(timestamp, ChronoUnit.MICROS); } - - /** - * Estimates the size of a single row in bytes for throughput calculation. - */ - private static int estimatedRowSize() { - // Rough estimate (binary protocol): - // - 2 symbols: ~10 bytes each = 20 bytes - // - 3 longs: 8 bytes each = 24 bytes - // - 4 doubles: 8 bytes each = 32 bytes - // - 1 string: ~10 bytes average - // - 1 boolean: 1 byte - // - 2 timestamps: 8 bytes each = 16 bytes - // - Overhead: ~20 bytes - // Total: ~123 bytes - return 123; - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index abbf41f..bed59a3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -61,36 +61,36 @@ */ public class StacBenchmarkClient { - private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; - private static final String PROTOCOL_ILP_HTTP = "ilp-http"; - private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; - - private static final String DEFAULT_HOST = "localhost"; - private static final int DEFAULT_ROWS = 80_000_000; private static final int DEFAULT_BATCH_SIZE = 10_000; private static final int DEFAULT_FLUSH_BYTES = 0; private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; + private static final String DEFAULT_HOST = "localhost"; private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; - private static final int DEFAULT_SEND_QUEUE = 0; - private static final int DEFAULT_WARMUP_ROWS = 100_000; private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; + private static final int DEFAULT_ROWS = 80_000_000; + private static final int DEFAULT_SEND_QUEUE = 0; private static final String DEFAULT_TABLE = "q"; - - // 8512 unique 4-letter symbols, as per STAC NYSE benchmark - private static final int SYMBOL_COUNT = 8512; - private static final String[] SYMBOLS = generateSymbols(SYMBOL_COUNT); - + private static final int DEFAULT_WARMUP_ROWS = 100_000; + // Estimated row size for throughput calculation: + // - 1 symbol: ~6 bytes (4-char + overhead) + // - 1 char: 2 bytes + // - 2 floats: 4 bytes each = 8 bytes + // - 2 shorts: 2 bytes each = 4 bytes + // - 1 boolean: 1 byte + // - 1 timestamp: 8 bytes + // - overhead: ~10 bytes + // Total: ~39 bytes + private static final int ESTIMATED_ROW_SIZE = 39; // Exchange codes (single characters) private static final char[] EXCHANGES = {'N', 'Q', 'A', 'B', 'C', 'D', 'P', 'Z'}; // Pre-computed single-char strings to avoid allocation private static final String[] EXCHANGE_STRINGS = new String[EXCHANGES.length]; - - static { - for (int i = 0; i < EXCHANGES.length; i++) { - EXCHANGE_STRINGS[i] = String.valueOf(EXCHANGES[i]); - } - } - + private static final String PROTOCOL_ILP_HTTP = "ilp-http"; + private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; + private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; + // 8512 unique 4-letter symbols, as per STAC NYSE benchmark + private static final int SYMBOL_COUNT = 8512; + private static final String[] SYMBOLS = generateSymbols(SYMBOL_COUNT); // Pre-computed bid base prices per symbol (to generate realistic spreads) private static final float[] BASE_PRICES = generateBasePrices(SYMBOL_COUNT); @@ -176,6 +176,80 @@ public static void main(String[] args) { } } + private static Sender createSender(String protocol, String host, int port, + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, int sendQueue) { + switch (protocol) { + case PROTOCOL_ILP_TCP: + return Sender.builder(Sender.Transport.TCP) + .address(host) + .port(port) + .build(); + case PROTOCOL_ILP_HTTP: + return Sender.builder(Sender.Transport.HTTP) + .address(host) + .port(port) + .autoFlushRows(batchSize) + .build(); + case PROTOCOL_QWP_WEBSOCKET: + Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) + .address(host) + .port(port) + .asyncMode(true); + if (batchSize > 0) b.autoFlushRows(batchSize); + if (flushBytes > 0) b.autoFlushBytes(flushBytes); + if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); + if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); + if (sendQueue > 0) b.sendQueueCapacity(sendQueue); + return b.build(); + default: + throw new IllegalArgumentException("Unknown protocol: " + protocol + + ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); + } + } + + /** + * Generates pseudo-random base prices for each symbol. + * Prices range from $1 to $500 to simulate realistic stock prices. + */ + private static float[] generateBasePrices(int count) { + float[] prices = new float[count]; + Random rng = new Random(42); // fixed seed for reproducibility + for (int i = 0; i < count; i++) { + prices[i] = 1.0f + rng.nextFloat() * 499.0f; + } + return prices; + } + + /** + * Generates N unique 4-letter symbols. + * Uses combinations of uppercase letters to produce predictable, reproducible symbols. + */ + private static String[] generateSymbols(int count) { + String[] symbols = new String[count]; + int idx = 0; + // 26^4 = 456,976 possible 4-letter combinations, far more than 8512 + outer: + for (char a = 'A'; a <= 'Z' && idx < count; a++) { + for (char b = 'A'; b <= 'Z' && idx < count; b++) { + for (char c = 'A'; c <= 'Z' && idx < count; c++) { + for (char d = 'A'; d <= 'Z' && idx < count; d++) { + symbols[idx++] = new String(new char[]{a, b, c, d}); + if (idx >= count) break outer; + } + } + } + } + return symbols; + } + + private static int getDefaultPort(String protocol) { + if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { + return 9000; + } + return 9009; + } + private static void printUsage() { System.out.println("STAC Benchmark Ingestion Client"); System.out.println(); @@ -212,13 +286,6 @@ private static void printUsage() { System.out.println(" ) timestamp(T) PARTITION BY DAY WAL;"); } - private static int getDefaultPort(String protocol) { - if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { - return 9000; - } - return 9009; - } - private static void runTest(String protocol, String host, int port, String table, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, int inFlightWindow, int sendQueue, @@ -305,38 +372,6 @@ private static void runTest(String protocol, String host, int port, String table } } - private static Sender createSender(String protocol, String host, int port, - int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { - switch (protocol) { - case PROTOCOL_ILP_TCP: - return Sender.builder(Sender.Transport.TCP) - .address(host) - .port(port) - .build(); - case PROTOCOL_ILP_HTTP: - return Sender.builder(Sender.Transport.HTTP) - .address(host) - .port(port) - .autoFlushRows(batchSize) - .build(); - case PROTOCOL_QWP_WEBSOCKET: - Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) - .address(host) - .port(port) - .asyncMode(true); - if (batchSize > 0) b.autoFlushRows(batchSize); - if (flushBytes > 0) b.autoFlushBytes(flushBytes); - if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); - if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); - return b.build(); - default: - throw new IllegalArgumentException("Unknown protocol: " + protocol + - ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); - } - } - /** * Sends a single quote row matching the STAC schema. *

@@ -376,49 +411,9 @@ private static void sendQuoteRow(Sender sender, String table, int rowIndex) { .at(timestamp, ChronoUnit.MICROS); } - /** - * Generates N unique 4-letter symbols. - * Uses combinations of uppercase letters to produce predictable, reproducible symbols. - */ - private static String[] generateSymbols(int count) { - String[] symbols = new String[count]; - int idx = 0; - // 26^4 = 456,976 possible 4-letter combinations, far more than 8512 - outer: - for (char a = 'A'; a <= 'Z' && idx < count; a++) { - for (char b = 'A'; b <= 'Z' && idx < count; b++) { - for (char c = 'A'; c <= 'Z' && idx < count; c++) { - for (char d = 'A'; d <= 'Z' && idx < count; d++) { - symbols[idx++] = new String(new char[]{a, b, c, d}); - if (idx >= count) break outer; - } - } - } - } - return symbols; - } - - /** - * Generates pseudo-random base prices for each symbol. - * Prices range from $1 to $500 to simulate realistic stock prices. - */ - private static float[] generateBasePrices(int count) { - float[] prices = new float[count]; - Random rng = new Random(42); // fixed seed for reproducibility - for (int i = 0; i < count; i++) { - prices[i] = 1.0f + rng.nextFloat() * 499.0f; + static { + for (int i = 0; i < EXCHANGES.length; i++) { + EXCHANGE_STRINGS[i] = String.valueOf(EXCHANGES[i]); } - return prices; } - - // Estimated row size for throughput calculation: - // - 1 symbol: ~6 bytes (4-char + overhead) - // - 1 char: 2 bytes - // - 2 floats: 4 bytes each = 8 bytes - // - 2 shorts: 2 bytes each = 4 bytes - // - 1 boolean: 1 byte - // - 1 timestamp: 8 bytes - // - overhead: ~10 bytes - // Total: ~39 bytes - private static final int ESTIMATED_ROW_SIZE = 39; -} \ No newline at end of file +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java index 023cf13..f31956b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -46,86 +46,126 @@ public class InFlightWindowTest { @Test - public void testBasicAddAndAcknowledge() { + public void testAcknowledgeAlreadyAcked() { InFlightWindow window = new InFlightWindow(8, 1000); - assertTrue(window.isEmpty()); - assertEquals(0, window.getInFlightCount()); - - // Add a batch (sequential: 0) window.addInFlight(0); - assertFalse(window.isEmpty()); - assertEquals(1, window.getInFlightCount()); + window.addInFlight(1); - // Acknowledge it (cumulative ACK up to 0) + // ACK up to 1 + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + + // ACK for already acknowledged sequence returns true (idempotent) assertTrue(window.acknowledge(0)); + assertTrue(window.acknowledge(1)); assertTrue(window.isEmpty()); - assertEquals(0, window.getInFlightCount()); - assertEquals(1, window.getTotalAcked()); } @Test - public void testMultipleBatches() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testAcknowledgeUpToAllBatches() { + InFlightWindow window = new InFlightWindow(16, 1000); - // Add sequential batches 0-4 - for (long i = 0; i < 5; i++) { + // Add batches + for (int i = 0; i < 10; i++) { window.addInFlight(i); } - assertEquals(5, window.getInFlightCount()); - // Cumulative ACK up to 2 (acknowledges 0, 1, 2) - assertEquals(3, window.acknowledgeUpTo(2)); - assertEquals(2, window.getInFlightCount()); + // ACK all with high sequence + int acked = window.acknowledgeUpTo(Long.MAX_VALUE); + assertEquals(10, acked); + assertTrue(window.isEmpty()); + } - // Cumulative ACK up to 4 (acknowledges 3, 4) - assertEquals(2, window.acknowledgeUpTo(4)); + @Test + public void testAcknowledgeUpToBasic() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches 0-9 + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + assertEquals(10, window.getInFlightCount()); + + // ACK up to 5 (should remove 0-5, leaving 6-9) + int acked = window.acknowledgeUpTo(5); + assertEquals(6, acked); + assertEquals(4, window.getInFlightCount()); + assertEquals(6, window.getTotalAcked()); + } + + @Test + public void testAcknowledgeUpToEmpty() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // ACK on empty window should be no-op + assertEquals(0, window.acknowledgeUpTo(100)); assertTrue(window.isEmpty()); - assertEquals(5, window.getTotalAcked()); } @Test - public void testAcknowledgeAlreadyAcked() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testAcknowledgeUpToIdempotent() { + InFlightWindow window = new InFlightWindow(16, 1000); window.addInFlight(0); window.addInFlight(1); + window.addInFlight(2); - // ACK up to 1 - assertTrue(window.acknowledge(1)); + // First ACK + assertEquals(3, window.acknowledgeUpTo(2)); assertTrue(window.isEmpty()); - // ACK for already acknowledged sequence returns true (idempotent) - assertTrue(window.acknowledge(0)); - assertTrue(window.acknowledge(1)); + // Duplicate ACK - should be no-op + assertEquals(0, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // ACK with lower sequence - should be no-op + assertEquals(0, window.acknowledgeUpTo(1)); assertTrue(window.isEmpty()); } @Test - public void testWindowFull() { - InFlightWindow window = new InFlightWindow(3, 1000); + public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(16, 5000); - // Fill the window window.addInFlight(0); window.addInFlight(1); window.addInFlight(2); - assertTrue(window.isFull()); - assertEquals(3, window.getInFlightCount()); + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); - // Free slots by ACKing - window.acknowledgeUpTo(1); - assertFalse(window.isFull()); - assertEquals(1, window.getInFlightCount()); + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertTrue(waiting.get()); + + // Single cumulative ACK clears all + window.acknowledgeUpTo(2); + + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + assertTrue(window.isEmpty()); } @Test - public void testWindowBlocksWhenFull() throws Exception { - InFlightWindow window = new InFlightWindow(2, 5000); + public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(3, 5000); // Fill the window window.addInFlight(0); window.addInFlight(1); + window.addInFlight(2); + assertTrue(window.isFull()); AtomicBoolean blocked = new AtomicBoolean(true); CountDownLatch started = new CountDownLatch(1); @@ -134,44 +174,23 @@ public void testWindowBlocksWhenFull() throws Exception { // Start thread that will block Thread addThread = new Thread(() -> { started.countDown(); - window.addInFlight(2); + window.addInFlight(3); blocked.set(false); finished.countDown(); }); addThread.start(); - // Wait for thread to start and block assertTrue(started.await(1, TimeUnit.SECONDS)); Thread.sleep(100); // Give time to block assertTrue(blocked.get()); - // Free a slot - window.acknowledge(0); + // Cumulative ACK frees multiple slots + window.acknowledgeUpTo(1); // Removes 0 and 1 // Thread should complete assertTrue(finished.await(1, TimeUnit.SECONDS)); assertFalse(blocked.get()); - assertEquals(2, window.getInFlightCount()); - } - - @Test - public void testWindowBlocksTimeout() { - InFlightWindow window = new InFlightWindow(2, 100); // 100ms timeout - - // Fill the window - window.addInFlight(0); - window.addInFlight(1); - - // Try to add another - should timeout - long start = System.currentTimeMillis(); - try { - window.addInFlight(2); - fail("Expected timeout exception"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Timeout")); - } - long elapsed = System.currentTimeMillis() - start; - assertTrue("Should have waited at least 100ms", elapsed >= 90); + assertEquals(2, window.getInFlightCount()); // batch 2 and 3 } @Test @@ -205,23 +224,6 @@ public void testAwaitEmpty() throws Exception { assertFalse(waiting.get()); } - @Test - public void testAwaitEmptyTimeout() { - InFlightWindow window = new InFlightWindow(8, 100); // 100ms timeout - - window.addInFlight(0); - - long start = System.currentTimeMillis(); - try { - window.awaitEmpty(); - fail("Expected timeout exception"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Timeout")); - } - long elapsed = System.currentTimeMillis() - start; - assertTrue("Should have waited at least 100ms", elapsed >= 90); - } - @Test public void testAwaitEmptyAlreadyEmpty() { InFlightWindow window = new InFlightWindow(8, 1000); @@ -232,58 +234,39 @@ public void testAwaitEmptyAlreadyEmpty() { } @Test - public void testFailBatch() { - InFlightWindow window = new InFlightWindow(8, 1000); - - window.addInFlight(0); - window.addInFlight(1); - - // Fail batch 0 - RuntimeException error = new RuntimeException("Test error"); - window.fail(0, error); - - assertEquals(1, window.getTotalFailed()); - assertNotNull(window.getLastError()); - } - - @Test - public void testFailPropagatesError() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testAwaitEmptyTimeout() { + InFlightWindow window = new InFlightWindow(8, 100); // 100ms timeout window.addInFlight(0); - window.fail(0, new RuntimeException("Test error")); - - // Subsequent operations should throw - try { - window.addInFlight(1); - fail("Expected exception due to error"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); - } + long start = System.currentTimeMillis(); try { window.awaitEmpty(); - fail("Expected exception due to error"); + fail("Expected timeout exception"); } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); + assertTrue(e.getMessage().contains("Timeout")); } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); } @Test - public void testFailAllPropagatesError() { + public void testBasicAddAndAcknowledge() { InFlightWindow window = new InFlightWindow(8, 1000); + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + + // Add a batch (sequential: 0) window.addInFlight(0); - window.addInFlight(1); - window.failAll(new RuntimeException("Transport down")); + assertFalse(window.isEmpty()); + assertEquals(1, window.getInFlightCount()); - try { - window.awaitEmpty(); - fail("Expected exception due to failAll"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); - assertTrue(e.getMessage().contains("Transport down")); - } + // Acknowledge it (cumulative ACK up to 0) + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + assertEquals(1, window.getTotalAcked()); } @Test @@ -303,21 +286,6 @@ public void testClearError() { assertEquals(2, window.getInFlightCount()); // 0 and 1 both in window (fail doesn't remove) } - @Test - public void testReset() { - InFlightWindow window = new InFlightWindow(8, 1000); - - window.addInFlight(0); - window.addInFlight(1); - window.fail(2, new RuntimeException("Test")); - - window.reset(); - - assertTrue(window.isEmpty()); - assertNull(window.getLastError()); - assertEquals(0, window.getInFlightCount()); - } - @Test public void testConcurrentAddAndAck() throws Exception { InFlightWindow window = new InFlightWindow(4, 5000); @@ -371,37 +339,138 @@ public void testConcurrentAddAndAck() throws Exception { } @Test - public void testFailWakesBlockedAdder() throws Exception { - InFlightWindow window = new InFlightWindow(2, 5000); - - // Fill the window - window.addInFlight(0); - window.addInFlight(1); - - CountDownLatch started = new CountDownLatch(1); - AtomicReference caught = new AtomicReference<>(); + public void testConcurrentAddAndCumulativeAck() throws Exception { + InFlightWindow window = new InFlightWindow(100, 10000); + int numBatches = 500; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); - // Thread that will block on add - Thread addThread = new Thread(() -> { - started.countDown(); + // Sender thread + Thread sender = new Thread(() -> { try { - window.addInFlight(2); - } catch (LineSenderException e) { - caught.set(e); + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); } }); - addThread.start(); - assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Let it block + // ACK thread using cumulative ACKs + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } else { + Thread.sleep(1); + } + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); - // Fail a batch - should wake the blocked thread + sender.start(); + acker.start(); + + assertTrue(done.await(30, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } + + @Test + public void testDefaultWindowSize() { + InFlightWindow window = new InFlightWindow(); + assertEquals(InFlightWindow.DEFAULT_WINDOW_SIZE, window.getMaxWindowSize()); + } + + @Test + public void testFailAllPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.failAll(new RuntimeException("Transport down")); + + try { + window.awaitEmpty(); + fail("Expected exception due to failAll"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + assertTrue(e.getMessage().contains("Transport down")); + } + } + + @Test + public void testFailBatch() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // Fail batch 0 + RuntimeException error = new RuntimeException("Test error"); + window.fail(0, error); + + assertEquals(1, window.getTotalFailed()); + assertNotNull(window.getLastError()); + } + + @Test + public void testFailPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); window.fail(0, new RuntimeException("Test error")); - addThread.join(1000); - assertFalse(addThread.isAlive()); - assertNotNull(caught.get()); - assertTrue(caught.get().getMessage().contains("failed")); + // Subsequent operations should throw + try { + window.addInFlight(1); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + try { + window.awaitEmpty(); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + } + + @Test + public void testFailThenClearThenAdd() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Error")); + + // Should not be able to add + try { + window.addInFlight(1); + fail("Expected exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + // Clear error + window.clearError(); + + // Should work now + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); } @Test @@ -436,29 +505,38 @@ public void testFailWakesAwaitEmpty() throws Exception { assertTrue(caught.get().getMessage().contains("failed")); } - @Test(expected = IllegalArgumentException.class) - public void testInvalidWindowSize() { - new InFlightWindow(0, 1000); - } - @Test - public void testGetMaxWindowSize() { - InFlightWindow window = new InFlightWindow(16, 1000); - assertEquals(16, window.getMaxWindowSize()); - } + public void testFailWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); - @Test - public void testRapidAddAndAck() { - InFlightWindow window = new InFlightWindow(16, 5000); + // Fill the window + window.addInFlight(0); + window.addInFlight(1); - // Rapid add and ack in same thread - for (int i = 0; i < 10000; i++) { - window.addInFlight(i); - assertTrue(window.acknowledge(i)); - } + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); - assertTrue(window.isEmpty()); - assertEquals(10000, window.getTotalAcked()); + // Thread that will block on add + Thread addThread = new Thread(() -> { + started.countDown(); + try { + window.addInFlight(2); + } catch (LineSenderException e) { + caught.set(e); + } + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Let it block + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + addThread.join(1000); + assertFalse(addThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); } @Test @@ -484,267 +562,154 @@ public void testFillAndDrainRepeatedly() { } @Test - public void testMultipleResets() { - InFlightWindow window = new InFlightWindow(8, 1000); - - for (int cycle = 0; cycle < 10; cycle++) { - window.addInFlight(cycle); - window.reset(); - - assertTrue(window.isEmpty()); - assertNull(window.getLastError()); - } + public void testGetMaxWindowSize() { + InFlightWindow window = new InFlightWindow(16, 1000); + assertEquals(16, window.getMaxWindowSize()); } @Test - public void testFailThenClearThenAdd() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testHasWindowSpace() { + InFlightWindow window = new InFlightWindow(2, 1000); + assertTrue(window.hasWindowSpace()); window.addInFlight(0); - window.fail(0, new RuntimeException("Error")); - - // Should not be able to add - try { - window.addInFlight(1); - fail("Expected exception"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); - } - - // Clear error - window.clearError(); - - // Should work now + assertTrue(window.hasWindowSpace()); window.addInFlight(1); - assertEquals(2, window.getInFlightCount()); - } - - @Test - public void testDefaultWindowSize() { - InFlightWindow window = new InFlightWindow(); - assertEquals(InFlightWindow.DEFAULT_WINDOW_SIZE, window.getMaxWindowSize()); - } - - @Test - public void testSmallestPossibleWindow() { - InFlightWindow window = new InFlightWindow(1, 1000); - - window.addInFlight(0); - assertTrue(window.isFull()); + assertFalse(window.hasWindowSpace()); window.acknowledge(0); - assertFalse(window.isFull()); + assertTrue(window.hasWindowSpace()); } @Test - public void testVeryLargeWindow() { - InFlightWindow window = new InFlightWindow(10000, 1000); + public void testHighConcurrencyStress() throws Exception { + InFlightWindow window = new InFlightWindow(8, 30000); + int numBatches = 10000; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); - // Add many batches - for (int i = 0; i < 5000; i++) { - window.addInFlight(i); - } - assertEquals(5000, window.getInFlightCount()); - assertFalse(window.isFull()); + // Fast sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); - // ACK half - window.acknowledgeUpTo(2499); - assertEquals(2500, window.getInFlightCount()); - } - - @Test - public void testZeroBatchId() { - InFlightWindow window = new InFlightWindow(8, 1000); + // Fast ACK thread + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } + // No sleep - maximum contention + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); - window.addInFlight(0); - assertEquals(1, window.getInFlightCount()); + sender.start(); + acker.start(); - assertTrue(window.acknowledge(0)); + assertTrue(done.await(60, TimeUnit.SECONDS)); + if (error.get() != null) { + error.get().printStackTrace(); + } + assertNull(error.get()); assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); } - // ==================== CUMULATIVE ACK TESTS ==================== + @Test(expected = IllegalArgumentException.class) + public void testInvalidWindowSize() { + new InFlightWindow(0, 1000); + } @Test - public void testAcknowledgeUpToBasic() { - InFlightWindow window = new InFlightWindow(16, 1000); + public void testMultipleBatches() { + InFlightWindow window = new InFlightWindow(8, 1000); - // Add batches 0-9 - for (int i = 0; i < 10; i++) { + // Add sequential batches 0-4 + for (long i = 0; i < 5; i++) { window.addInFlight(i); } - assertEquals(10, window.getInFlightCount()); - - // ACK up to 5 (should remove 0-5, leaving 6-9) - int acked = window.acknowledgeUpTo(5); - assertEquals(6, acked); - assertEquals(4, window.getInFlightCount()); - assertEquals(6, window.getTotalAcked()); - } - - @Test - public void testAcknowledgeUpToIdempotent() { - InFlightWindow window = new InFlightWindow(16, 1000); - - window.addInFlight(0); - window.addInFlight(1); - window.addInFlight(2); + assertEquals(5, window.getInFlightCount()); - // First ACK + // Cumulative ACK up to 2 (acknowledges 0, 1, 2) assertEquals(3, window.acknowledgeUpTo(2)); - assertTrue(window.isEmpty()); - - // Duplicate ACK - should be no-op - assertEquals(0, window.acknowledgeUpTo(2)); - assertTrue(window.isEmpty()); + assertEquals(2, window.getInFlightCount()); - // ACK with lower sequence - should be no-op - assertEquals(0, window.acknowledgeUpTo(1)); + // Cumulative ACK up to 4 (acknowledges 3, 4) + assertEquals(2, window.acknowledgeUpTo(4)); assertTrue(window.isEmpty()); + assertEquals(5, window.getTotalAcked()); } @Test - public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { - InFlightWindow window = new InFlightWindow(3, 5000); - - // Fill the window - window.addInFlight(0); - window.addInFlight(1); - window.addInFlight(2); - assertTrue(window.isFull()); - - AtomicBoolean blocked = new AtomicBoolean(true); - CountDownLatch started = new CountDownLatch(1); - CountDownLatch finished = new CountDownLatch(1); - - // Start thread that will block - Thread addThread = new Thread(() -> { - started.countDown(); - window.addInFlight(3); - blocked.set(false); - finished.countDown(); - }); - addThread.start(); - - assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Give time to block - assertTrue(blocked.get()); + public void testMultipleResets() { + InFlightWindow window = new InFlightWindow(8, 1000); - // Cumulative ACK frees multiple slots - window.acknowledgeUpTo(1); // Removes 0 and 1 + for (int cycle = 0; cycle < 10; cycle++) { + window.addInFlight(cycle); + window.reset(); - // Thread should complete - assertTrue(finished.await(1, TimeUnit.SECONDS)); - assertFalse(blocked.get()); - assertEquals(2, window.getInFlightCount()); // batch 2 and 3 + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + } } @Test - public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { + public void testRapidAddAndAck() { InFlightWindow window = new InFlightWindow(16, 5000); - window.addInFlight(0); - window.addInFlight(1); - window.addInFlight(2); - - AtomicBoolean waiting = new AtomicBoolean(true); - CountDownLatch started = new CountDownLatch(1); - CountDownLatch finished = new CountDownLatch(1); - - // Start thread waiting for empty - Thread waitThread = new Thread(() -> { - started.countDown(); - window.awaitEmpty(); - waiting.set(false); - finished.countDown(); - }); - waitThread.start(); - - assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); - assertTrue(waiting.get()); - - // Single cumulative ACK clears all - window.acknowledgeUpTo(2); + // Rapid add and ack in same thread + for (int i = 0; i < 10000; i++) { + window.addInFlight(i); + assertTrue(window.acknowledge(i)); + } - assertTrue(finished.await(1, TimeUnit.SECONDS)); - assertFalse(waiting.get()); assertTrue(window.isEmpty()); + assertEquals(10000, window.getTotalAcked()); } @Test - public void testAcknowledgeUpToEmpty() { - InFlightWindow window = new InFlightWindow(16, 1000); - - // ACK on empty window should be no-op - assertEquals(0, window.acknowledgeUpTo(100)); - assertTrue(window.isEmpty()); - } + public void testReset() { + InFlightWindow window = new InFlightWindow(8, 1000); - @Test - public void testAcknowledgeUpToAllBatches() { - InFlightWindow window = new InFlightWindow(16, 1000); + window.addInFlight(0); + window.addInFlight(1); + window.fail(2, new RuntimeException("Test")); - // Add batches - for (int i = 0; i < 10; i++) { - window.addInFlight(i); - } + window.reset(); - // ACK all with high sequence - int acked = window.acknowledgeUpTo(Long.MAX_VALUE); - assertEquals(10, acked); assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + assertEquals(0, window.getInFlightCount()); } @Test - public void testConcurrentAddAndCumulativeAck() throws Exception { - InFlightWindow window = new InFlightWindow(100, 10000); - int numBatches = 500; - CountDownLatch done = new CountDownLatch(2); - AtomicReference error = new AtomicReference<>(); - AtomicInteger highestAdded = new AtomicInteger(-1); - - // Sender thread - Thread sender = new Thread(() -> { - try { - for (int i = 0; i < numBatches; i++) { - window.addInFlight(i); - highestAdded.set(i); - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } - }); - - // ACK thread using cumulative ACKs - Thread acker = new Thread(() -> { - try { - int lastAcked = -1; - while (lastAcked < numBatches - 1) { - int highest = highestAdded.get(); - if (highest > lastAcked) { - window.acknowledgeUpTo(highest); - lastAcked = highest; - } else { - Thread.sleep(1); - } - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } - }); + public void testSmallestPossibleWindow() { + InFlightWindow window = new InFlightWindow(1, 1000); - sender.start(); - acker.start(); + window.addInFlight(0); + assertTrue(window.isFull()); - assertTrue(done.await(30, TimeUnit.SECONDS)); - assertNull(error.get()); - assertTrue(window.isEmpty()); - assertEquals(numBatches, window.getTotalAcked()); + window.acknowledge(0); + assertFalse(window.isFull()); } @Test @@ -764,69 +729,102 @@ public void testTryAddInFlight() { } @Test - public void testHasWindowSpace() { - InFlightWindow window = new InFlightWindow(2, 1000); + public void testVeryLargeWindow() { + InFlightWindow window = new InFlightWindow(10000, 1000); - assertTrue(window.hasWindowSpace()); + // Add many batches + for (int i = 0; i < 5000; i++) { + window.addInFlight(i); + } + assertEquals(5000, window.getInFlightCount()); + assertFalse(window.isFull()); + + // ACK half + window.acknowledgeUpTo(2499); + assertEquals(2500, window.getInFlightCount()); + } + + @Test + public void testWindowBlocksTimeout() { + InFlightWindow window = new InFlightWindow(2, 100); // 100ms timeout + + // Fill the window window.addInFlight(0); - assertTrue(window.hasWindowSpace()); window.addInFlight(1); - assertFalse(window.hasWindowSpace()); - window.acknowledge(0); - assertTrue(window.hasWindowSpace()); + // Try to add another - should timeout + long start = System.currentTimeMillis(); + try { + window.addInFlight(2); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); } @Test - public void testHighConcurrencyStress() throws Exception { - InFlightWindow window = new InFlightWindow(8, 30000); - int numBatches = 10000; - CountDownLatch done = new CountDownLatch(2); - AtomicReference error = new AtomicReference<>(); - AtomicInteger highestAdded = new AtomicInteger(-1); + public void testWindowBlocksWhenFull() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); - // Fast sender thread - Thread sender = new Thread(() -> { - try { - for (int i = 0; i < numBatches; i++) { - window.addInFlight(i); - highestAdded.set(i); - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } - }); + // Fill the window + window.addInFlight(0); + window.addInFlight(1); - // Fast ACK thread - Thread acker = new Thread(() -> { - try { - int lastAcked = -1; - while (lastAcked < numBatches - 1) { - int highest = highestAdded.get(); - if (highest > lastAcked) { - window.acknowledgeUpTo(highest); - lastAcked = highest; - } - // No sleep - maximum contention - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(2); + blocked.set(false); + finished.countDown(); }); + addThread.start(); - sender.start(); - acker.start(); + // Wait for thread to start and block + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Give time to block + assertTrue(blocked.get()); - assertTrue(done.await(60, TimeUnit.SECONDS)); - if (error.get() != null) { - error.get().printStackTrace(); - } - assertNull(error.get()); + // Free a slot + window.acknowledge(0); + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testWindowFull() { + InFlightWindow window = new InFlightWindow(3, 1000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + assertTrue(window.isFull()); + assertEquals(3, window.getInFlightCount()); + + // Free slots by ACKing + window.acknowledgeUpTo(1); + assertFalse(window.isFull()); + assertEquals(1, window.getInFlightCount()); + } + + @Test + public void testZeroBatchId() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + assertEquals(1, window.getInFlightCount()); + + assertTrue(window.acknowledge(0)); assertTrue(window.isEmpty()); - assertEquals(numBatches, window.getTotalAcked()); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 4b31040..ecb2bb7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -67,8 +67,6 @@ public void testAddressNull_fails() { () -> Sender.builder(Sender.Transport.WEBSOCKET).address(null)); } - // ==================== Transport Selection Tests ==================== - @Test public void testAddressWithoutPort_usesDefaultPort9000() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -85,8 +83,6 @@ public void testAsyncModeCanBeSetMultipleTimes() { Assert.assertNotNull(builder); } - // ==================== Address Configuration Tests ==================== - @Test public void testAsyncModeDisabled() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -166,8 +162,6 @@ public void testAutoFlushIntervalMillisDoubleSet_fails() { .autoFlushIntervalMillis(200)); } - // ==================== TLS Configuration Tests ==================== - @Test public void testAutoFlushIntervalMillisNegative_fails() { assertThrows("cannot be negative", @@ -209,8 +203,6 @@ public void testAutoFlushRowsNegative_fails() { .autoFlushRows(-1)); } - // ==================== Async Mode Tests ==================== - @Test public void testAutoFlushRowsZero_disablesRowBasedAutoFlush() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -236,8 +228,6 @@ public void testBufferCapacityDoubleSet_fails() { .bufferCapacity(2048)); } - // ==================== Auto Flush Rows Tests ==================== - @Test public void testBufferCapacityNegative_fails() { assertThrows("cannot be negative", @@ -275,8 +265,6 @@ public void testConnectionRefused() throws Exception { ); } - // ==================== Auto Flush Bytes Tests ==================== - @Test public void testCustomTrustStore_butTlsNotEnabled_fails() { assertThrowsAny( @@ -312,8 +300,6 @@ public void testDuplicateAddresses_fails() { .address(LOCALHOST + ":9000")); } - // ==================== Auto Flush Interval Tests ==================== - @Test @Ignore("TCP authentication is not supported for WebSocket protocol") public void testEnableAuth_notSupported() { @@ -361,8 +347,6 @@ public void testHttpPath_mayNotApply() { Assert.assertNotNull(builder); } - // ==================== In-Flight Window Size Tests ==================== - @Test @Ignore("HTTP timeout is HTTP-specific and may not apply to WebSocket") public void testHttpTimeout_mayNotApply() { @@ -410,8 +394,6 @@ public void testInFlightWindowSizeNegative_fails() { .inFlightWindowSize(-1)); } - // ==================== Send Queue Capacity Tests ==================== - @Test public void testInFlightWindowSizeZero_fails() { assertThrows("must be positive", @@ -450,8 +432,6 @@ public void testInvalidSchema_fails() { assertBadConfig("invalid::addr=localhost:9000;", "invalid schema [schema=invalid, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); } - // ==================== Combined Async Configuration Tests ==================== - @Test public void testMalformedPortInAddress_fails() { assertThrows("cannot parse a port from the address", @@ -467,8 +447,6 @@ public void testMaxBackoff_mayNotApply() { Assert.assertNotNull(builder); } - // ==================== Config String Tests (ws:// and wss://) ==================== - @Test public void testMaxNameLength() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -528,8 +506,6 @@ public void testPortMismatch_fails() { "mismatch"); } - // ==================== Buffer Configuration Tests ==================== - @Test @Ignore("Protocol version is for ILP text protocol, WebSocket uses ILP v4 binary protocol") public void testProtocolVersion_notApplicable() { @@ -558,8 +534,6 @@ public void testSendQueueCapacityDoubleSet_fails() { .sendQueueCapacity(32)); } - // ==================== Unsupported Features (TCP Authentication) ==================== - @Test public void testSendQueueCapacityNegative_fails() { assertThrows("must be positive", @@ -578,8 +552,6 @@ public void testSendQueueCapacityZero_fails() { .sendQueueCapacity(0)); } - // ==================== Unsupported Features (HTTP Token Authentication) ==================== - @Test public void testSendQueueCapacity_withAsyncMode() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -598,30 +570,6 @@ public void testSendQueueCapacity_withoutAsyncMode_fails() { "requires async mode"); } - // ==================== Unsupported Features (Username/Password Authentication) ==================== - - @Test - public void testSyncModeDoesNotAllowInFlightWindowSize() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false) - .inFlightWindowSize(16), - "requires async mode"); - } - - @Test - public void testSyncModeDoesNotAllowSendQueueCapacity() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false) - .sendQueueCapacity(32), - "requires async mode"); - } - - // ==================== Unsupported Features (HTTP-specific options) ==================== - @Test public void testSyncModeAutoFlushDefaults() throws Exception { // Regression test: sync-mode connect() must not hardcode autoFlush to 0. @@ -648,6 +596,26 @@ public void testSyncModeAutoFlushDefaults() throws Exception { }); } + @Test + public void testSyncModeDoesNotAllowInFlightWindowSize() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .inFlightWindowSize(16), + "requires async mode"); + } + + @Test + public void testSyncModeDoesNotAllowSendQueueCapacity() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .sendQueueCapacity(32), + "requires async mode"); + } + @Test public void testSyncModeIsDefault() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -699,8 +667,6 @@ public void testTlsValidationDisabled_butTlsNotEnabled_fails() { "TLS was not enabled"); } - // ==================== Unsupported Features (Protocol Version) ==================== - @Test public void testUsernamePassword_fails() { assertThrowsAny( @@ -710,8 +676,6 @@ public void testUsernamePassword_fails() { "not yet supported"); } - // ==================== Config String Unsupported Options ==================== - @Test @Ignore("Username/password authentication is not yet supported for WebSocket protocol") public void testUsernamePassword_notYetSupported() { @@ -728,8 +692,6 @@ public void testWsConfigString() throws Exception { assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); } - // ==================== Edge Cases ==================== - @Test public void testWsConfigString_missingAddr_fails() throws Exception { int port = findUnusedPort(); @@ -764,8 +726,6 @@ public void testWsConfigString_withUsernamePassword_notYetSupported() { assertBadConfig("ws::addr=localhost:9000;username=user;password=pass;", "not yet supported"); } - // ==================== Connection Tests ==================== - @Test public void testWssConfigString() { assertBadConfig("wss::addr=localhost:9000;tls_verify=unsafe_off;", "connect", "Failed", "SSL"); @@ -776,8 +736,6 @@ public void testWssConfigString_uppercaseNotSupported() { assertBadConfig("WSS::addr=localhost:9000;", "invalid schema"); } - // ==================== Sync vs Async Mode Tests ==================== - @SuppressWarnings("resource") private static void assertBadConfig(String config, String... anyOf) { assertThrowsAny(() -> Sender.fromConfig(config), anyOf); @@ -796,12 +754,6 @@ private static void assertThrowsAny(Sender.LineSenderBuilder builder, String... assertThrowsAny(builder::build, anyOf); } - private static int findUnusedPort() throws Exception { - try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { - return s.getLocalPort(); - } - } - private static void assertThrowsAny(Runnable action, String... anyOf) { try { action.run(); @@ -816,4 +768,10 @@ private static void assertThrowsAny(Runnable action, String... anyOf) { Assert.fail("Expected message containing one of [" + String.join(", ", anyOf) + "] but got: " + msg); } } + + private static int findUnusedPort() throws Exception { + try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { + return s.getLocalPort(); + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java index cbc81ef..a1f1ecf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -32,7 +32,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class MicrobatchBufferTest { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 46eae77..f51aa54 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -33,6 +33,15 @@ public class NativeBufferWriterTest { + @Test + public void testEnsureCapacityGrowsBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + assertEquals(16, writer.getCapacity()); + writer.ensureCapacity(32); + assertTrue(writer.getCapacity() >= 32); + } + } + @Test public void testPatchIntAtLastValidOffset() { try (NativeBufferWriter writer = new NativeBufferWriter(16)) { @@ -77,13 +86,4 @@ public void testSkipThenPatchInt() { assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); } } - - @Test - public void testEnsureCapacityGrowsBuffer() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - assertEquals(16, writer.getCapacity()); - writer.ensureCapacity(32); - assertTrue(writer.getCapacity() >= 32); - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index 02106f0..0d0d157 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -26,9 +26,9 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; -import io.questdb.client.std.Decimal64; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; import org.junit.Assert; import org.junit.BeforeClass; @@ -54,412 +54,304 @@ public static void setUpStatic() { AbstractLineSenderTest.setUpStatic(); } - // === BYTE coercion tests === - @Test - public void testByteToBooleanCoercionError() throws Exception { - String table = "test_qwp_byte_to_boolean_error"; + public void testBoolToString() throws Exception { + String table = "test_qwp_bool_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("b", (byte) 1) + .boolColumn("s", true) .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning BYTE and BOOLEAN but got: " + msg, - msg.contains("BYTE") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testByteToCharCoercionError() throws Exception { - String table = "test_qwp_byte_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("c", (byte) 65) - .at(1_000_000, ChronoUnit.MICROS); + .boolColumn("s", false) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning BYTE and CHAR but got: " + msg, - msg.contains("BYTE") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDate() throws Exception { - String table = "test_qwp_byte_to_date"; + public void testBoolToVarchar() throws Exception { + String table = "test_qwp_bool_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 100) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("d", (byte) 0) + .boolColumn("v", false) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal() throws Exception { - String table = "test_qwp_byte_to_decimal"; + public void testBoolean() throws Exception { + String table = "test_qwp_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("b", true) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("d", (byte) -100) + .boolColumn("b", false) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "b\ttimestamp\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testByteToDecimal128() throws Exception { - String table = "test_qwp_byte_to_decimal128"; + public void testBooleanToByteCoercionError() throws Exception { + String table = "test_qwp_boolean_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("BYTE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal16() throws Exception { - String table = "test_qwp_byte_to_decimal16"; + public void testBooleanToCharCoercionError() throws Exception { + String table = "test_qwp_boolean_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal256() throws Exception { - String table = "test_qwp_byte_to_decimal256"; + public void testBooleanToDateCoercionError() throws Exception { + String table = "test_qwp_boolean_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DATE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal64() throws Exception { - String table = "test_qwp_byte_to_decimal64"; + public void testBooleanToDecimalCoercionError() throws Exception { + String table = "test_qwp_boolean_to_decimal_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DECIMAL") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal8() throws Exception { - String table = "test_qwp_byte_to_decimal8"; + public void testBooleanToDoubleCoercionError() throws Exception { + String table = "test_qwp_boolean_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 5) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DOUBLE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDouble() throws Exception { - String table = "test_qwp_byte_to_double"; + public void testBooleanToFloatCoercionError() throws Exception { + String table = "test_qwp_boolean_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("FLOAT") + ); + } } @Test - public void testByteToFloat() throws Exception { - String table = "test_qwp_byte_to_float"; + public void testBooleanToGeoHashCoercionError() throws Exception { + String table = "test_qwp_boolean_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("f", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("f", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("GEOHASH") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToGeoHashCoercionError() throws Exception { - String table = "test_qwp_byte_to_geohash_error"; + public void testBooleanToIntCoercionError() throws Exception { + String table = "test_qwp_boolean_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("g", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning BYTE but got: " + msg, - msg.contains("type coercion from BYTE to") && msg.contains("is not supported") + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("INT") ); } } @Test - public void testByteToInt() throws Exception { - String table = "test_qwp_byte_to_int"; + public void testBooleanToLong256CoercionError() throws Exception { + String table = "test_qwp_boolean_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("i", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("i", Byte.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("i", Byte.MIN_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG256") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "127\t1970-01-01T00:00:02.000000000Z\n" + - "-128\t1970-01-01T00:00:03.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToLong() throws Exception { - String table = "test_qwp_byte_to_long"; + public void testBooleanToLongCoercionError() throws Exception { + String table = "test_qwp_boolean_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("l", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("l", Byte.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("l", Byte.MIN_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "127\t1970-01-01T00:00:02.000000000Z\n" + - "-128\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToLong256CoercionError() throws Exception { - String table = "test_qwp_byte_to_long256_error"; + public void testBooleanToShortCoercionError() throws Exception { + String table = "test_qwp_boolean_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("v", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -467,375 +359,383 @@ public void testByteToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from BYTE to LONG256 is not supported") + msg.contains("cannot write BOOLEAN") && msg.contains("SHORT") ); } } @Test - public void testByteToShort() throws Exception { - String table = "test_qwp_byte_to_short"; + public void testBooleanToSymbolCoercionError() throws Exception { + String table = "test_qwp_boolean_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testBooleanToTimestampCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", Byte.MIN_VALUE) - .at(2_000_000, ChronoUnit.MICROS); + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testBooleanToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", Byte.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); } + } - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + @Test + public void testBooleanToUuidCoercionError() throws Exception { + String table = "test_qwp_boolean_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("UUID") + ); + } } @Test - public void testByteToString() throws Exception { - String table = "test_qwp_byte_to_string"; + public void testByte() throws Exception { + String table = "test_qwp_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", (byte) 42) + .shortColumn("b", (short) -1) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("s", (byte) -100) + .shortColumn("b", (short) 0) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("s", (byte) 0) + .shortColumn("b", (short) 127) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToSymbol() throws Exception { - String table = "test_qwp_byte_to_symbol"; + public void testByteToBooleanCoercionError() throws Exception { + String table = "test_qwp_byte_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", (byte) 42) + .byteColumn("b", (byte) 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToTimestamp() throws Exception { - String table = "test_qwp_byte_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("t", (byte) 100) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("t", (byte) 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning BYTE and BOOLEAN but got: " + msg, + msg.contains("BYTE") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToUuidCoercionError() throws Exception { - String table = "test_qwp_byte_to_uuid_error"; + public void testByteToCharCoercionError() throws Exception { + String table = "test_qwp_byte_to_char_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "c CHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("u", (byte) 42) + .byteColumn("c", (byte) 65) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from BYTE to UUID is not supported") + "Expected error mentioning BYTE and CHAR but got: " + msg, + msg.contains("BYTE") && msg.contains("CHAR") ); } } @Test - public void testByteToVarchar() throws Exception { - String table = "test_qwp_byte_to_varchar"; + public void testByteToDate() throws Exception { + String table = "test_qwp_byte_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("v", (byte) 42) + .byteColumn("d", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("v", (byte) -100) + .byteColumn("d", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("v", Byte.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } - // === Exact Type Match Tests === - @Test - public void testBoolean() throws Exception { - String table = "test_qwp_boolean"; + public void testByteToDecimal() throws Exception { + String table = "test_qwp_byte_to_decimal"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("b", true) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .boolColumn("b", false) + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\ttimestamp\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByte() throws Exception { - String table = "test_qwp_byte"; + public void testByteToDecimal128() throws Exception { + String table = "test_qwp_byte_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) -1) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("b", (short) 0) + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testChar() throws Exception { - String table = "test_qwp_char"; + public void testByteToDecimal16() throws Exception { + String table = "test_qwp_byte_to_decimal16"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("c", 'A') + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", 'ü') // ü + .byteColumn("d", (byte) -9) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("c", '中') // 中 - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "c\ttimestamp\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "ü\t1970-01-01T00:00:02.000000000Z\n" + - "中\t1970-01-01T00:00:03.000000000Z\n", - "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal() throws Exception { - String table = "test_qwp_decimal"; + public void testByteToDecimal256() throws Exception { + String table = "test_qwp_byte_to_decimal256"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", "123.45") + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", "-999.99") + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", "0.01") - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(42_000, 2)) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDouble() throws Exception { - String table = "test_qwp_double"; + public void testByteToDecimal64() throws Exception { + String table = "test_qwp_byte_to_decimal64"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 42.5) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -1.0E10) + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MIN_VALUE) - .at(4_000_000, ChronoUnit.MICROS); - // NaN and Inf should be stored as null - sender.table(table) - .doubleColumn("d", Double.NaN) - .at(5_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.POSITIVE_INFINITY) - .at(6_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.NEGATIVE_INFINITY) - .at(7_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 7); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\ttimestamp\n" + - "42.5\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + - "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + - "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + - "null\t1970-01-01T00:00:05.000000000Z\n" + - "null\t1970-01-01T00:00:06.000000000Z\n" + - "null\t1970-01-01T00:00:07.000000000Z\n", - "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleArray() throws Exception { - String table = "test_qwp_double_array"; + public void testByteToDecimal8() throws Exception { + String table = "test_qwp_byte_to_decimal8"; useTable(table); - - double[] arr1d = createDoubleArray(5); - double[][] arr2d = createDoubleArray(2, 3); - double[][][] arr3d = createDoubleArray(1, 2, 3); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("a1", arr1d) - .doubleArray("a2", arr2d) - .doubleArray("a3", arr3d) + .byteColumn("d", (byte) 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("d", (byte) -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimal() throws Exception { - String table = "test_qwp_double_to_decimal"; + public void testByteToDouble() throws Exception { + String table = "test_qwp_byte_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 123.45) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -42.10) + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -843,221 +743,188 @@ public void testDoubleToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_decimal_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("d", 123.456) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("cannot be converted to") && msg.contains("123.456") && msg.contains("scale=2") - ); - } - } - - @Test - public void testDoubleToByte() throws Exception { - String table = "test_qwp_double_to_byte"; + public void testByteToFloat() throws Exception { + String table = "test_qwp_byte_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 42.0) + .byteColumn("f", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("b", -100.0) + .byteColumn("f", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToBytePrecisionLossError() throws Exception { - String table = "test_qwp_double_to_byte_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("b", 42.5) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("42.5") - ); - } + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToByteOverflowError() throws Exception { - String table = "test_qwp_double_to_byte_ovf"; + public void testByteToGeoHashCoercionError() throws Exception { + String table = "test_qwp_byte_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 200.0) + .byteColumn("g", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 200 out of range for BYTE") + "Expected coercion error mentioning BYTE but got: " + msg, + msg.contains("type coercion from BYTE to") && msg.contains("is not supported") ); } } @Test - public void testDoubleToFloat() throws Exception { - String table = "test_qwp_double_to_float"; + public void testByteToInt() throws Exception { + String table = "test_qwp_byte_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("f", 1.5) + .byteColumn("i", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("f", -42.25) + .byteColumn("i", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("i", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToInt() throws Exception { - String table = "test_qwp_double_to_int"; + public void testByteToLong() throws Exception { + String table = "test_qwp_byte_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 100_000.0) + .byteColumn("l", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("i", -42.0) + .byteColumn("l", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("l", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + - "100000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToIntPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_int_prec"; + public void testByteToLong256CoercionError() throws Exception { + String table = "test_qwp_byte_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 3.14) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("3.14") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to LONG256 is not supported") ); } } @Test - public void testDoubleToLong() throws Exception { - String table = "test_qwp_double_to_long"; + public void testByteToShort() throws Exception { + String table = "test_qwp_byte_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("l", 1_000_000.0) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("l", -42.0) + .byteColumn("s", Byte.MIN_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "1000000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToString() throws Exception { - String table = "test_qwp_double_to_string"; + public void testByteToString() throws Exception { + String table = "test_qwp_byte_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + "s STRING, " + @@ -1067,620 +934,589 @@ public void testDoubleToString() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("s", 3.14) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("s", -42.0) + .byteColumn("s", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "s\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToSymbol() throws Exception { - String table = "test_qwp_double_to_symbol"; + public void testByteToSymbol() throws Exception { + String table = "test_qwp_byte_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("sym", 3.14) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); assertSqlEventually( - "sym\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToVarchar() throws Exception { - String table = "test_qwp_double_to_varchar"; + public void testByteToTimestamp() throws Exception { + String table = "test_qwp_byte_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .byteColumn("t", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("v", -42.0) + .byteColumn("t", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloat() throws Exception { - String table = "test_qwp_float"; + public void testByteToUuidCoercionError() throws Exception { + String table = "test_qwp_byte_to_uuid_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("f", 1.5f) + .byteColumn("u", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("f", -42.25f) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("f", 0.0f) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to UUID is not supported") + ); } - - assertTableSizeEventually(table, 3); } @Test - public void testFloatToDecimal() throws Exception { - String table = "test_qwp_float_to_decimal"; + public void testByteToVarchar() throws Exception { + String table = "test_qwp_byte_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .byteColumn("v", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("v", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "1.50\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_decimal_prec"; + public void testChar() throws Exception { + String table = "test_qwp_char"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.25f) + .charColumn("c", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", 'ü') // ü + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '中') // 中 + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "c\ttimestamp\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "ü\t1970-01-01T00:00:02.000000000Z\n" + + "中\t1970-01-01T00:00:03.000000000Z\n", + "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testCharToBooleanCoercionError() throws Exception { + String table = "test_qwp_char_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("cannot be converted to") && msg.contains("scale=1") + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("BOOLEAN") ); } } @Test - public void testFloatToDouble() throws Exception { - String table = "test_qwp_float_to_double"; + public void testCharToByteCoercionError() throws Exception { + String table = "test_qwp_char_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("d", -42.25f) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("BYTE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToInt() throws Exception { - String table = "test_qwp_float_to_int"; + public void testCharToDateCoercionError() throws Exception { + String table = "test_qwp_char_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 42.0f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("i", -100.0f) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DATE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToIntPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_int_prec"; + public void testCharToDoubleCoercionError() throws Exception { + String table = "test_qwp_char_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 3.14f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DOUBLE") ); } } @Test - public void testFloatToLong() throws Exception { - String table = "test_qwp_float_to_long"; + public void testCharToFloatCoercionError() throws Exception { + String table = "test_qwp_char_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("l", 1000.0f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("FLOAT") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "l\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToString() throws Exception { - String table = "test_qwp_float_to_string"; + public void testCharToGeoHashCoercionError() throws Exception { + String table = "test_qwp_char_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("s", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("GEOHASH") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "s\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToSymbol() throws Exception { - String table = "test_qwp_float_to_symbol"; + public void testCharToIntCoercionError() throws Exception { + String table = "test_qwp_char_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("sym", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("INT") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "sym\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToVarchar() throws Exception { - String table = "test_qwp_float_to_varchar"; + public void testCharToLong256CoercionError() throws Exception { + String table = "test_qwp_char_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG256") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "v\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testInt() throws Exception { - String table = "test_qwp_int"; + public void testCharToLongCoercionError() throws Exception { + String table = "test_qwp_char_to_long_error"; useTable(table); - + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .intColumn("i", Integer.MIN_VALUE) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", -42) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); - } - - assertTableSizeEventually(table, 4); - assertSqlEventually( - "i\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n" + - "-42\t1970-01-01T00:00:04.000000000Z\n", - "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG") + ); + } } @Test - public void testIntToBooleanCoercionError() throws Exception { - String table = "test_qwp_int_to_boolean_error"; + public void testCharToShortCoercionError() throws Exception { + String table = "test_qwp_char_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 1) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and BOOLEAN but got: " + msg, - msg.contains("INT") && msg.contains("BOOLEAN") + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("SHORT") ); } } @Test - public void testIntToByte() throws Exception { - String table = "test_qwp_int_to_byte"; + public void testCharToString() throws Exception { + String table = "test_qwp_char_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 42) + .charColumn("s", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("b", -128) + .charColumn("s", 'Z') .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToByteOverflowError() throws Exception { - String table = "test_qwp_int_to_byte_overflow"; + public void testCharToSymbolCoercionError() throws Exception { + String table = "test_qwp_char_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 128) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("SYMBOL") ); } } @Test - public void testIntToCharCoercionError() throws Exception { - String table = "test_qwp_int_to_char_error"; + public void testCharToUuidCoercionError() throws Exception { + String table = "test_qwp_char_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("c", 65) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and CHAR but got: " + msg, - msg.contains("INT") && msg.contains("CHAR") + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("UUID") ); } } @Test - public void testIntToDate() throws Exception { - String table = "test_qwp_int_to_date"; + public void testCharToVarchar() throws Exception { + String table = "test_qwp_char_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 86_400_000 millis = 1 day sender.table(table) - .intColumn("d", 86_400_000) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", 0) + .charColumn("v", 'Z') .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal() throws Exception { - String table = "test_qwp_int_to_decimal"; + public void testDecimal() throws Exception { + String table = "test_qwp_decimal"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", "-999.99") .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "0.01") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(42_000, 2)) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 4); } @Test - public void testIntToDecimal128() throws Exception { - String table = "test_qwp_int_to_decimal128"; + public void testDecimal128ToDecimal256() throws Exception { + String table = "test_qwp_dec128_to_dec256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal16() throws Exception { - String table = "test_qwp_int_to_decimal16"; + public void testDecimal128ToDecimal64() throws Exception { + String table = "test_qwp_dec128_to_dec64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal256() throws Exception { - String table = "test_qwp_int_to_decimal256"; + public void testDecimal256ToDecimal128() throws Exception { + String table = "test_qwp_dec256_to_dec128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal64() throws Exception { - String table = "test_qwp_int_to_decimal64"; + public void testDecimal256ToDecimal64() throws Exception { + String table = "test_qwp_dec256_to_dec64"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(18, 2), " + @@ -1689,189 +1525,173 @@ public void testIntToDecimal64() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL256 wire type to DECIMAL64 column sender.table(table) - .intColumn("d", Integer.MAX_VALUE) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal8() throws Exception { - String table = "test_qwp_int_to_decimal8"; + public void testDecimal256ToDecimal64OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec64_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Create a value that fits in Decimal256 but overflows Decimal64 + // Decimal256 with hi bits set will overflow 64-bit storage + Decimal256 bigValue = Decimal256.fromBigDecimal(new java.math.BigDecimal("99999999999999999999.99")); sender.table(table) - .intColumn("d", 5) + .decimalColumn("d", bigValue) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -9) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDouble() throws Exception { - String table = "test_qwp_int_to_double"; + public void testDecimal256ToDecimal8OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec8_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 999.9 with scale=1 → unscaled 9999, which doesn't fit in a byte (-128..127) sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal256.fromLong(9999, 1)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToFloat() throws Exception { - String table = "test_qwp_int_to_float"; + public void testDecimal64ToDecimal128() throws Exception { + String table = "test_qwp_dec64_to_dec128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL64 wire type to DECIMAL128 column (widening) sender.table(table) - .intColumn("f", 42) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("f", -100) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("f", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToGeoHashCoercionError() throws Exception { - String table = "test_qwp_int_to_geohash_error"; + public void testDecimal64ToDecimal256() throws Exception { + String table = "test_qwp_dec64_to_dec256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("g", 42) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error mentioning INT but got: " + msg, - msg.contains("type coercion from INT to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToLong() throws Exception { - String table = "test_qwp_int_to_long"; + public void testDecimalRescale() throws Exception { + String table = "test_qwp_decimal_rescale"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "d DECIMAL(18, 4), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Send scale=2 wire data to scale=4 column: server should rescale sender.table(table) - .intColumn("l", 42) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("l", Integer.MAX_VALUE) + .decimalColumn("d", Decimal64.fromLong(-100, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", -1) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "2147483647\t1970-01-01T00:00:02.000000000Z\n" + - "-1\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "123.4500\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToLong256CoercionError() throws Exception { - String table = "test_qwp_int_to_long256_error"; + public void testDecimalToBooleanCoercionError() throws Exception { + String table = "test_qwp_decimal_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("v", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1879,174 +1699,125 @@ public void testIntToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to LONG256 is not supported") + msg.contains("cannot write DECIMAL64") && msg.contains("BOOLEAN") ); } } @Test - public void testIntToShort() throws Exception { - String table = "test_qwp_int_to_short"; + public void testDecimalToByteCoercionError() throws Exception { + String table = "test_qwp_decimal_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 1000) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -32768) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 32767) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("BYTE") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToShortOverflowError() throws Exception { - String table = "test_qwp_int_to_short_overflow"; + public void testDecimalToCharCoercionError() throws Exception { + String table = "test_qwp_decimal_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 32768) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("CHAR") ); } } @Test - public void testIntToString() throws Exception { - String table = "test_qwp_int_to_string"; + public void testDecimalToDateCoercionError() throws Exception { + String table = "test_qwp_decimal_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DATE") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToSymbol() throws Exception { - String table = "test_qwp_int_to_symbol"; + public void testDecimalToDoubleCoercionError() throws Exception { + String table = "test_qwp_decimal_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DOUBLE") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToTimestamp() throws Exception { - String table = "test_qwp_int_to_timestamp"; + public void testDecimalToFloatCoercionError() throws Exception { + String table = "test_qwp_decimal_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - // 1_000_000 micros = 1 second sender.table(table) - .intColumn("t", 1_000_000) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("t", 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("FLOAT") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToUuidCoercionError() throws Exception { - String table = "test_qwp_int_to_uuid_error"; + public void testDecimalToGeoHashCoercionError() throws Exception { + String table = "test_qwp_decimal_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("u", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -2054,517 +1825,446 @@ public void testIntToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to UUID is not supported") + msg.contains("cannot write DECIMAL64") && msg.contains("GEOHASH") ); } } @Test - public void testIntToVarchar() throws Exception { - String table = "test_qwp_int_to_varchar"; + public void testDecimalToIntCoercionError() throws Exception { + String table = "test_qwp_decimal_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("v", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("INT") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong() throws Exception { - String table = "test_qwp_long"; + public void testDecimalToLong256CoercionError() throws Exception { + String table = "test_qwp_decimal_to_long256_error"; useTable(table); - + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .longColumn("l", Long.MIN_VALUE) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", Long.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG256") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "l\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLong256() throws Exception { - String table = "test_qwp_long256"; + public void testDecimalToLongCoercionError() throws Exception { + String table = "test_qwp_decimal_to_long_error"; useTable(table); - + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // All zeros sender.table(table) - .long256Column("v", 0, 0, 0, 0) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - // Mixed values - sender.table(table) - .long256Column("v", 1, 2, 3, 4) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG") + ); } - - assertTableSizeEventually(table, 2); } @Test - public void testLongToBooleanCoercionError() throws Exception { - String table = "test_qwp_long_to_boolean_error"; + public void testDecimalToShortCoercionError() throws Exception { + String table = "test_qwp_decimal_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 1) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and BOOLEAN but got: " + msg, - msg.contains("LONG") && msg.contains("BOOLEAN") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SHORT") ); } } @Test - public void testLongToByte() throws Exception { - String table = "test_qwp_long_to_byte"; + public void testDecimalToString() throws Exception { + String table = "test_qwp_decimal_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 42) + .decimalColumn("s", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("b", -128) + .decimalColumn("s", Decimal64.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToByteOverflowError() throws Exception { - String table = "test_qwp_long_to_byte_overflow"; + public void testDecimalToSymbolCoercionError() throws Exception { + String table = "test_qwp_decimal_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 128) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SYMBOL") ); } } @Test - public void testLongToCharCoercionError() throws Exception { - String table = "test_qwp_long_to_char_error"; + public void testDecimalToTimestampCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("c", 65) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and CHAR but got: " + msg, - msg.contains("LONG") && msg.contains("CHAR") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") ); } } @Test - public void testLongToDate() throws Exception { - String table = "test_qwp_long_to_date"; + public void testDecimalToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_ns_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DATE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 86_400_000L) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", 0L) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal() throws Exception { - String table = "test_qwp_long_to_decimal"; + public void testDecimalToUuidCoercionError() throws Exception { + String table = "test_qwp_decimal_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("UUID") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal128() throws Exception { - String table = "test_qwp_long_to_decimal128"; + public void testDecimalToVarchar() throws Exception { + String table = "test_qwp_decimal_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 1_000_000_000L) + .decimalColumn("v", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000L) + .decimalColumn("v", Decimal64.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal16() throws Exception { - String table = "test_qwp_long_to_decimal16"; + public void testDouble() throws Exception { + String table = "test_qwp_double"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .doubleColumn("d", 42.5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .doubleColumn("d", -1.0E10) .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); + sender.table(table) + .doubleColumn("d", Double.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MIN_VALUE) + .at(4_000_000, ChronoUnit.MICROS); + // NaN and Inf should be stored as null + sender.table(table) + .doubleColumn("d", Double.NaN) + .at(5_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.POSITIVE_INFINITY) + .at(6_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.NEGATIVE_INFINITY) + .at(7_000_000, ChronoUnit.MICROS); + sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 7); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "d\ttimestamp\n" + + "42.5\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + + "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + + "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + + "null\t1970-01-01T00:00:05.000000000Z\n" + + "null\t1970-01-01T00:00:06.000000000Z\n" + + "null\t1970-01-01T00:00:07.000000000Z\n", + "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLongToDecimal256() throws Exception { - String table = "test_qwp_long_to_decimal256"; + public void testDoubleArray() throws Exception { + String table = "test_qwp_double_array"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + + double[] arr1d = createDoubleArray(5); + double[][] arr2d = createDoubleArray(2, 3); + double[][][] arr3d = createDoubleArray(1, 2, 3); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", Long.MAX_VALUE) + .doubleArray("a1", arr1d) + .doubleArray("a2", arr2d) + .doubleArray("a3", arr3d) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -1_000_000_000_000L) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 1); } @Test - public void testLongToDecimal32() throws Exception { - String table = "test_qwp_long_to_decimal32"; + public void testDoubleArrayToIntCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("INT") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal8() throws Exception { - String table = "test_qwp_long_to_decimal8"; + public void testDoubleArrayToStringCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_string_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 5) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("STRING") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDouble() throws Exception { - String table = "test_qwp_long_to_double"; + public void testDoubleArrayToSymbolCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("SYMBOL") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToFloat() throws Exception { - String table = "test_qwp_long_to_float"; + public void testDoubleArrayToTimestampCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_timestamp_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("f", 42) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("f", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("TIMESTAMP") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long_to_geohash_error"; + public void testDoubleToBooleanCoercionError() throws Exception { + String table = "test_qwp_double_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("g", 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning LONG but got: " + msg, - msg.contains("type coercion from LONG to") && msg.contains("is not supported") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("BOOLEAN") ); } } @Test - public void testLongToInt() throws Exception { - String table = "test_qwp_long_to_int"; + public void testDoubleToByte() throws Exception { + String table = "test_qwp_double_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in INT range should succeed sender.table(table) - .longColumn("i", 42) + .doubleColumn("b", 42.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("i", -1) + .doubleColumn("b", -100.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + + "b\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToIntOverflowError() throws Exception { - String table = "test_qwp_long_to_int_overflow"; + public void testDoubleToByteOverflowError() throws Exception { + String table = "test_qwp_double_to_byte_ovf"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("i", (long) Integer.MAX_VALUE + 1) + .doubleColumn("b", 200.0) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -2572,182 +2272,163 @@ public void testLongToIntOverflowError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected overflow error but got: " + msg, - msg.contains("integer value 2147483648 out of range for INT") + msg.contains("integer value 200 out of range for BYTE") ); } } @Test - public void testLongToLong256CoercionError() throws Exception { - String table = "test_qwp_long_to_long256_error"; + public void testDoubleToBytePrecisionLossError() throws Exception { + String table = "test_qwp_double_to_byte_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v LONG256, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .doubleColumn("b", 42.5) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to LONG256 is not supported") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("42.5") ); } } @Test - public void testLongToShort() throws Exception { - String table = "test_qwp_long_to_short"; + public void testDoubleToCharCoercionError() throws Exception { + String table = "test_qwp_double_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - // Value in SHORT range should succeed sender.table(table) - .longColumn("s", 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); } @Test - public void testLongToShortOverflowError() throws Exception { - String table = "test_qwp_long_to_short_overflow"; + public void testDoubleToDateCoercionError() throws Exception { + String table = "test_qwp_double_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 32768) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") ); } } @Test - public void testLongToString() throws Exception { - String table = "test_qwp_long_to_string"; + public void testDoubleToDecimal() throws Exception { + String table = "test_qwp_double_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .doubleColumn("d", 123.45) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", Long.MAX_VALUE) + .doubleColumn("d", -42.10) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToSymbol() throws Exception { - String table = "test_qwp_long_to_symbol"; + public void testDoubleToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .doubleColumn("d", 123.456) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("cannot be converted to") && msg.contains("123.456") && msg.contains("scale=2") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToTimestamp() throws Exception { - String table = "test_qwp_long_to_timestamp"; + public void testDoubleToFloat() throws Exception { + String table = "test_qwp_double_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("t", 1_000_000L) + .doubleColumn("f", 1.5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("t", 0L) + .doubleColumn("f", -42.25) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToUuidCoercionError() throws Exception { - String table = "test_qwp_long_to_uuid_error"; + public void testDoubleToGeoHashCoercionError() throws Exception { + String table = "test_qwp_double_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("u", 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -2755,429 +2436,370 @@ public void testLongToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to UUID is not supported") + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") ); } } @Test - public void testLongToVarchar() throws Exception { - String table = "test_qwp_long_to_varchar"; + public void testDoubleToInt() throws Exception { + String table = "test_qwp_double_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .doubleColumn("i", 100_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("v", Long.MAX_VALUE) + .doubleColumn("i", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testMultipleRowsAndBatching() throws Exception { - String table = "test_qwp_multiple_rows"; - useTable(table); - - int rowCount = 1000; - try (QwpWebSocketSender sender = createQwpSender()) { - for (int i = 0; i < rowCount; i++) { - sender.table(table) - .symbol("sym", "s" + (i % 10)) - .longColumn("val", i) - .doubleColumn("dbl", i * 1.5) - .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); - } - sender.flush(); - } - - assertTableSizeEventually(table, rowCount); + "i\tts\n" + + "100000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShort() throws Exception { - String table = "test_qwp_short"; + public void testDoubleToIntPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_int_prec"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Short.MIN_VALUE is the null sentinel for SHORT sender.table(table) - .shortColumn("s", Short.MIN_VALUE) + .doubleColumn("i", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("3.14") + ); } - - assertTableSizeEventually(table, 3); } @Test - public void testShortToDecimal128() throws Exception { - String table = "test_qwp_short_to_decimal128"; + public void testDoubleToLong() throws Exception { + String table = "test_qwp_double_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", Short.MAX_VALUE) + .doubleColumn("l", 1_000_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", Short.MIN_VALUE) + .doubleColumn("l", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "32767.00\t1970-01-01T00:00:01.000000000Z\n" + - "-32768.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal16() throws Exception { - String table = "test_qwp_short_to_decimal16"; + public void testDoubleToLong256CoercionError() throws Exception { + String table = "test_qwp_double_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal256() throws Exception { - String table = "test_qwp_short_to_decimal256"; + public void testDoubleToShort() throws Exception { + String table = "test_qwp_double_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "v SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("v", 100.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .doubleColumn("v", -200.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "100\t1970-01-01T00:00:01.000000000Z\n" + + "-200\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal32() throws Exception { - String table = "test_qwp_short_to_decimal32"; + public void testDoubleToString() throws Exception { + String table = "test_qwp_double_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("s", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .doubleColumn("s", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal64() throws Exception { - String table = "test_qwp_short_to_decimal64"; + public void testDoubleToSymbol() throws Exception { + String table = "test_qwp_double_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("sym", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "sym\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal8() throws Exception { - String table = "test_qwp_short_to_decimal8"; + public void testDoubleToUuidCoercionError() throws Exception { + String table = "test_qwp_double_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 5) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToInt() throws Exception { - String table = "test_qwp_short_to_int"; + public void testDoubleToVarchar() throws Exception { + String table = "test_qwp_double_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("i", (short) 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("i", Short.MAX_VALUE) + .doubleColumn("v", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToLong() throws Exception { - String table = "test_qwp_short_to_long"; + public void testFloat() throws Exception { + String table = "test_qwp_float"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("l", (short) 42) + .floatColumn("f", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("l", Short.MAX_VALUE) + .floatColumn("f", -42.25f) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 3); } @Test - public void testShortToBooleanCoercionError() throws Exception { - String table = "test_qwp_short_to_boolean_error"; + public void testFloatToBooleanCoercionError() throws Exception { + String table = "test_qwp_float_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) 1) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning SHORT and BOOLEAN but got: " + msg, - msg.contains("SHORT") && msg.contains("BOOLEAN") + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("BOOLEAN") ); } } @Test - public void testShortToByte() throws Exception { - String table = "test_qwp_short_to_byte"; + public void testFloatToByte() throws Exception { + String table = "test_qwp_float_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "v BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) 42) + .floatColumn("v", 7.0f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("b", (short) -128) + .floatColumn("v", -100.0f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "7\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToByteOverflowError() throws Exception { - String table = "test_qwp_short_to_byte_overflow"; + public void testFloatToCharCoercionError() throws Exception { + String table = "test_qwp_float_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) 128) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("CHAR") ); } } @Test - public void testShortToCharCoercionError() throws Exception { - String table = "test_qwp_short_to_char_error"; + public void testFloatToDateCoercionError() throws Exception { + String table = "test_qwp_float_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("c", (short) 65) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning SHORT and CHAR but got: " + msg, - msg.contains("SHORT") && msg.contains("CHAR") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") ); } } @Test - public void testShortToDate() throws Exception { - String table = "test_qwp_short_to_date"; + public void testFloatToDecimal() throws Exception { + String table = "test_qwp_float_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 1000 millis = 1 second sender.table(table) - .shortColumn("d", (short) 1000) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) 0) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -3185,222 +2807,268 @@ public void testShortToDate() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "1.50\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDouble() throws Exception { - String table = "test_qwp_short_to_double"; + public void testFloatToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "d DECIMAL(10, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .floatColumn("d", 1.25f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("cannot be converted to") && msg.contains("scale=1") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToFloat() throws Exception { - String table = "test_qwp_short_to_float"; + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("f", (short) 42) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("f", (short) -100) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToGeoHashCoercionError() throws Exception { - String table = "test_qwp_short_to_geohash_error"; + public void testFloatToGeoHashCoercionError() throws Exception { + String table = "test_qwp_float_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("g", (short) 42) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning SHORT but got: " + msg, - msg.contains("type coercion from SHORT to") && msg.contains("is not supported") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") ); } } @Test - public void testShortToLong256CoercionError() throws Exception { - String table = "test_qwp_short_to_long256_error"; + public void testFloatToInt() throws Exception { + String table = "test_qwp_float_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v LONG256, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("v", (short) 42) + .floatColumn("i", 42.0f) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("i", -100.0f) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from SHORT to LONG256 is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToString() throws Exception { - String table = "test_qwp_short_to_string"; + public void testFloatToIntPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_int_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("s", (short) 42) + .floatColumn("i", 3.14f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - } - - assertTableSizeEventually(table, 3); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") + ); + } + } + + @Test + public void testFloatToLong() throws Exception { + String table = "test_qwp_float_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("l", 1000.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToSymbol() throws Exception { - String table = "test_qwp_short_to_symbol"; + public void testFloatToLong256CoercionError() throws Exception { + String table = "test_qwp_float_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToShort() throws Exception { + String table = "test_qwp_float_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "v SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("s", (short) 42) + .floatColumn("v", 42.0f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("s", (short) -1) + .floatColumn("v", -1000.0f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + + "v\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "-1000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToTimestamp() throws Exception { - String table = "test_qwp_short_to_timestamp"; + public void testFloatToString() throws Exception { + String table = "test_qwp_float_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("t", (short) 1000) + .floatColumn("s", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("t", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToUuidCoercionError() throws Exception { - String table = "test_qwp_short_to_uuid_error"; + public void testFloatToSymbol() throws Exception { + String table = "test_qwp_float_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("u", (short) 42) + .floatColumn("sym", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "sym\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToUuidCoercionError() throws Exception { + String table = "test_qwp_float_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -3408,14 +3076,14 @@ public void testShortToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from SHORT to UUID is not supported") + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") ); } } @Test - public void testShortToVarchar() throws Exception { - String table = "test_qwp_short_to_varchar"; + public void testFloatToVarchar() throws Exception { + String table = "test_qwp_float_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + "v VARCHAR, " + @@ -3425,342 +3093,313 @@ public void testShortToVarchar() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("v", (short) 42) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("v", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("v", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", + "1.5\t1970-01-01T00:00:01.000000000Z\n", "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testString() throws Exception { - String table = "test_qwp_string"; + public void testInt() throws Exception { + String table = "test_qwp_int"; useTable(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .stringColumn("s", "hello world") + .intColumn("i", Integer.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "non-ascii äöü") + .intColumn("i", 0) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "") + .intColumn("i", Integer.MAX_VALUE) .at(3_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", null) + .intColumn("i", -42) .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 4); assertSqlEventually( - "s\ttimestamp\n" + - "hello world\t1970-01-01T00:00:01.000000000Z\n" + - "non-ascii äöü\t1970-01-01T00:00:02.000000000Z\n" + - "\t1970-01-01T00:00:03.000000000Z\n" + - "null\t1970-01-01T00:00:04.000000000Z\n", - "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); + "i\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testStringToChar() throws Exception { - String table = "test_qwp_string_to_char"; + public void testIntToBooleanCoercionError() throws Exception { + String table = "test_qwp_int_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "c CHAR, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("c", "A") + .intColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("c", "Hello") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and BOOLEAN but got: " + msg, + msg.contains("INT") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "c\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "H\t1970-01-01T00:00:02.000000000Z\n", - "SELECT c, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToSymbol() throws Exception { - String table = "test_qwp_string_to_symbol"; + public void testIntToByte() throws Exception { + String table = "test_qwp_int_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("s", "hello") + .intColumn("b", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "world") + .intColumn("b", -128) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToUuid() throws Exception { - String table = "test_qwp_string_to_uuid"; + public void testIntToByteOverflowError() throws Exception { + String table = "test_qwp_int_to_byte_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") + .intColumn("b", 128) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "u\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", - "SELECT u, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbol() throws Exception { - String table = "test_qwp_symbol"; + public void testIntToCharCoercionError() throws Exception { + String table = "test_qwp_int_to_char_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("s", "alpha") + .intColumn("c", 65) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", "beta") - .at(2_000_000, ChronoUnit.MICROS); - // repeated value reuses dictionary entry - sender.table(table) - .symbol("s", "alpha") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and CHAR but got: " + msg, + msg.contains("INT") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\ttimestamp\n" + - "alpha\t1970-01-01T00:00:01.000000000Z\n" + - "beta\t1970-01-01T00:00:02.000000000Z\n" + - "alpha\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testTimestampMicros() throws Exception { - String table = "test_qwp_timestamp_micros"; + public void testIntToDate() throws Exception { + String table = "test_qwp_int_to_date"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + // 86_400_000 millis = 1 day sender.table(table) - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .intColumn("d", 86_400_000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "ts_col\ttimestamp\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, timestamp FROM " + table); + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampMicrosToNanos() throws Exception { - String table = "test_qwp_timestamp_micros_to_nanos"; + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP_NS, " + + "d DECIMAL(6, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_111_111L; // 2022-02-25T00:00:00Z sender.table(table) - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - // Microseconds scaled to nanoseconds - assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, ts FROM " + table); - } - - @Test - public void testTimestampNanos() throws Exception { - String table = "test_qwp_timestamp_nanos"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsNanos = 1_645_747_200_000_000_000L; // 2022-02-25T00:00:00Z in nanos sender.table(table) - .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) - .at(tsNanos, ChronoUnit.NANOS); + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampNanosToMicros() throws Exception { - String table = "test_qwp_timestamp_nanos_to_micros"; + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsNanos = 1_645_747_200_123_456_789L; sender.table(table) - .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); - // Nanoseconds truncated to microseconds + assertTableSizeEventually(table, 3); assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, ts FROM " + table); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuid() throws Exception { - String table = "test_qwp_uuid"; + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; useTable(table); - - UUID uuid1 = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .uuidColumn("u", uuid1.getLeastSignificantBits(), uuid1.getMostSignificantBits()) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .uuidColumn("u", uuid2.getLeastSignificantBits(), uuid2.getMostSignificantBits()) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "u\ttimestamp\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + - "11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z\n", - "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToShortCoercionError() throws Exception { - String table = "test_qwp_uuid_to_short_error"; + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to SHORT is not supported") - ); - } - } - - @Test - public void testWriteAllTypesInOneRow() throws Exception { - String table = "test_qwp_all_types"; - useTable(table); - - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - double[] arr1d = {1.0, 2.0, 3.0}; - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z - - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("sym", "test_symbol") - .boolColumn("bool_col", true) - .shortColumn("short_col", (short) 42) - .intColumn("int_col", 100_000) - .longColumn("long_col", 1_000_000_000L) - .floatColumn("float_col", 2.5f) - .doubleColumn("double_col", 3.14) - .stringColumn("string_col", "hello") - .charColumn("char_col", 'Z') - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) - .uuidColumn("uuid_col", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .long256Column("long256_col", 1, 0, 0, 0) - .doubleArray("arr_col", arr1d) - .decimalColumn("decimal_col", "99.99") - .at(tsMicros, ChronoUnit.MICROS); + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } - // === Decimal cross-width coercion tests === - @Test - public void testDecimal256ToDecimal64() throws Exception { - String table = "test_qwp_dec256_to_dec64"; + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(18, 2), " + @@ -3769,69 +3408,75 @@ public void testDecimal256ToDecimal64() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Send DECIMAL256 wire type to DECIMAL64 column sender.table(table) - .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .intColumn("d", Integer.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal256ToDecimal128() throws Exception { - String table = "test_qwp_dec256_to_dec128"; + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .intColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .intColumn("d", -9) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal64ToDecimal128() throws Exception { - String table = "test_qwp_dec64_to_dec128"; + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Send DECIMAL64 wire type to DECIMAL128 column (widening) sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -3839,166 +3484,170 @@ public void testDecimal64ToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal64ToDecimal256() throws Exception { - String table = "test_qwp_dec64_to_dec256"; + public void testIntToFloat() throws Exception { + String table = "test_qwp_int_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .intColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .intColumn("f", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal128ToDecimal64() throws Exception { - String table = "test_qwp_dec128_to_dec64"; + public void testIntToGeoHashCoercionError() throws Exception { + String table = "test_qwp_int_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .intColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal128.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning INT but got: " + msg, + msg.contains("type coercion from INT to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal128ToDecimal256() throws Exception { - String table = "test_qwp_dec128_to_dec256"; + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .intColumn("l", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal128.fromLong(-9999, 2)) + .intColumn("l", Integer.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalRescale() throws Exception { - String table = "test_qwp_decimal_rescale"; + public void testIntToLong256CoercionError() throws Exception { + String table = "test_qwp_int_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 4), " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Send scale=2 wire data to scale=4 column: server should rescale sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-100, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to LONG256 is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.4500\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0000\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal256ToDecimal64OverflowError() throws Exception { - String table = "test_qwp_dec256_to_dec64_overflow"; + public void testIntToShort() throws Exception { + String table = "test_qwp_int_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Create a value that fits in Decimal256 but overflows Decimal64 - // Decimal256 with hi bits set will overflow 64-bit storage - Decimal256 bigValue = Decimal256.fromBigDecimal(new java.math.BigDecimal("99999999999999999999.99")); sender.table(table) - .decimalColumn("d", bigValue) + .intColumn("s", 1000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -32768) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 32767) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("decimal value overflows") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal256ToDecimal8OverflowError() throws Exception { - String table = "test_qwp_dec256_to_dec8_overflow"; + public void testIntToShortOverflowError() throws Exception { + String table = "test_qwp_int_to_short_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 999.9 with scale=1 → unscaled 9999, which doesn't fit in a byte (-128..127) sender.table(table) - .decimalColumn("d", Decimal256.fromLong(9999, 1)) + .intColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -4006,580 +3655,422 @@ public void testDecimal256ToDecimal8OverflowError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected overflow error but got: " + msg, - msg.contains("decimal value overflows") + msg.contains("integer value 32768 out of range for SHORT") ); } } @Test - public void testStringToBoolean() throws Exception { - String table = "test_qwp_string_to_boolean"; + public void testIntToString() throws Exception { + String table = "test_qwp_int_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "true") + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", "false") + .intColumn("s", -100) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", "1") + .intColumn("s", 0) .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "0") - .at(4_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "TRUE") - .at(5_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 5); + assertTableSizeEventually(table, 3); assertSqlEventually( - "b\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n" + - "true\t1970-01-01T00:00:03.000000000Z\n" + - "false\t1970-01-01T00:00:04.000000000Z\n" + - "true\t1970-01-01T00:00:05.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToBooleanParseError() throws Exception { - String table = "test_qwp_string_to_boolean_err"; + public void testIntToSymbol() throws Exception { + String table = "test_qwp_int_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "yes") + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse boolean from string") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToByte() throws Exception { - String table = "test_qwp_string_to_byte"; + public void testIntToTimestamp() throws Exception { + String table = "test_qwp_int_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 1_000_000 micros = 1 second sender.table(table) - .stringColumn("b", "42") + .intColumn("t", 1_000_000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", "-128") + .intColumn("t", 0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "127") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToByteParseError() throws Exception { - String table = "test_qwp_string_to_byte_err"; + public void testIntToUuidCoercionError() throws Exception { + String table = "test_qwp_int_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "abc") + .intColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse BYTE from string") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to UUID is not supported") ); } } @Test - public void testStringToDate() throws Exception { - String table = "test_qwp_string_to_date"; + public void testIntToVarchar() throws Exception { + String table = "test_qwp_int_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "2022-02-25T00:00:00.000Z") + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("v", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("v", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal64() throws Exception { - String table = "test_qwp_string_to_dec64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal128() throws Exception { - String table = "test_qwp_string_to_dec128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal256() throws Exception { - String table = "test_qwp_string_to_dec256"; + public void testLong() throws Exception { + String table = "test_qwp_long"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .stringColumn("d", "123.45") + .longColumn("l", Long.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-99.99") + .longColumn("l", 0) .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDouble() throws Exception { - String table = "test_qwp_string_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "3.14") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-2.718") - .at(2_000_000, ChronoUnit.MICROS); + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-2.718\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testStringToFloat() throws Exception { - String table = "test_qwp_string_to_float"; + public void testLong256() throws Exception { + String table = "test_qwp_long256"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros sender.table(table) - .stringColumn("f", "3.14") + .long256Column("v", 0, 0, 0, 0) .at(1_000_000, ChronoUnit.MICROS); + // Mixed values sender.table(table) - .stringColumn("f", "-2.5") + .long256Column("v", 1, 2, 3, 4) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-2.5\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToGeoHash() throws Exception { - String table = "test_qwp_string_to_geohash"; + public void testLong256ToBooleanCoercionError() throws Exception { + String table = "test_qwp_long256_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(5c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("g", "s24se") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("g", "u33dc") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "g\tts\n" + - "s24se\t1970-01-01T00:00:01.000000000Z\n" + - "u33dc\t1970-01-01T00:00:02.000000000Z\n", - "SELECT g, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToInt() throws Exception { - String table = "test_qwp_string_to_int"; + public void testLong256ToByteCoercionError() throws Exception { + String table = "test_qwp_long256_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("i", "42") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("i", "-100") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("i", "0") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToLong() throws Exception { - String table = "test_qwp_string_to_long"; + public void testLong256ToCharCoercionError() throws Exception { + String table = "test_qwp_long256_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("l", "1000000000000") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("l", "-1") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "l\tts\n" + - "1000000000000\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToLong256() throws Exception { - String table = "test_qwp_string_to_long256"; + public void testLong256ToDateCoercionError() throws Exception { + String table = "test_qwp_long256_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("l", "0x01") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "l\tts\n" + - "0x01\t1970-01-01T00:00:01.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToShort() throws Exception { - String table = "test_qwp_string_to_short"; + public void testLong256ToDoubleCoercionError() throws Exception { + String table = "test_qwp_long256_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("s", "1000") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "-32768") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "32767") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToTimestamp() throws Exception { - String table = "test_qwp_string_to_timestamp"; + public void testLong256ToFloatCoercionError() throws Exception { + String table = "test_qwp_long256_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "t\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBoolToString() throws Exception { - String table = "test_qwp_bool_to_string"; + public void testLong256ToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long256_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("s", true) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("s", false) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBoolToVarchar() throws Exception { - String table = "test_qwp_bool_to_varchar"; + public void testLong256ToIntCoercionError() throws Exception { + String table = "test_qwp_long256_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("v", false) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalToString() throws Exception { - String table = "test_qwp_decimal_to_string"; + public void testLong256ToLongCoercionError() throws Exception { + String table = "test_qwp_long256_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("s", Decimal64.fromLong(12345, 2)) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("s", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalToVarchar() throws Exception { - String table = "test_qwp_decimal_to_varchar"; + public void testLong256ToShortCoercionError() throws Exception { + String table = "test_qwp_long256_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345, 2)) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToString() throws Exception { - String table = "test_qwp_symbol_to_string"; + public void testLong256ToString() throws Exception { + String table = "test_qwp_long256_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + "s STRING, " + @@ -4589,78 +4080,63 @@ public void testSymbolToString() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("s", "hello") + .long256Column("s", 1, 2, 3, 4) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", "world") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( "s\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); } @Test - public void testSymbolToVarchar() throws Exception { - String table = "test_qwp_symbol_to_varchar"; + public void testLong256ToSymbolCoercionError() throws Exception { + String table = "test_qwp_long256_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("v", "world") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("SYMBOL") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampToString() throws Exception { - String table = "test_qwp_timestamp_to_string"; + public void testLong256ToUuidCoercionError() throws Exception { + String table = "test_qwp_long256_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .timestampColumn("s", tsMicros, ChronoUnit.MICROS) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "s\tts\n" + - "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampToVarchar() throws Exception { - String table = "test_qwp_timestamp_to_varchar"; + public void testLong256ToVarchar() throws Exception { + String table = "test_qwp_long256_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + "v VARCHAR, " + @@ -4669,9 +4145,8 @@ public void testTimestampToVarchar() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .timestampColumn("v", tsMicros, ChronoUnit.MICROS) + .long256Column("v", 1, 2, 3, 4) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4679,214 +4154,217 @@ public void testTimestampToVarchar() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); } @Test - public void testCharToString() throws Exception { - String table = "test_qwp_char_to_string"; + public void testLongToBooleanCoercionError() throws Exception { + String table = "test_qwp_long_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("s", 'A') + .longColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("s", 'Z') - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and BOOLEAN but got: " + msg, + msg.contains("LONG") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToVarchar() throws Exception { - String table = "test_qwp_char_to_varchar"; + public void testLongToByte() throws Exception { + String table = "test_qwp_long_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .longColumn("b", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("v", 'Z') + .longColumn("b", -128) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "v\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToShort() throws Exception { - String table = "test_qwp_double_to_short"; + public void testLongToByteOverflowError() throws Exception { + String table = "test_qwp_long_to_byte_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v SHORT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 100.0) + .longColumn("b", 128) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("v", -200.0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "100\t1970-01-01T00:00:01.000000000Z\n" + - "-200\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToByte() throws Exception { - String table = "test_qwp_float_to_byte"; + public void testLongToCharCoercionError() throws Exception { + String table = "test_qwp_long_to_char_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v BYTE, " + + "c CHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 7.0f) + .longColumn("c", 65) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("v", -100.0f) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and CHAR but got: " + msg, + msg.contains("LONG") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "7\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToShort() throws Exception { - String table = "test_qwp_float_to_short"; + public void testLongToDate() throws Exception { + String table = "test_qwp_long_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v SHORT, " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 42.0f) + .longColumn("d", 86_400_000L) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("v", -1000.0f) + .longColumn("d", 0L) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1000\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToString() throws Exception { - String table = "test_qwp_long256_to_string"; + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("s", 1, 2, 3, 4) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToVarchar() throws Exception { - String table = "test_qwp_long256_to_varchar"; + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1, 2, 3, 4) + .longColumn("d", 1_000_000_000L) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table); + "d\tts\n" + + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal8() throws Exception { - String table = "test_qwp_string_to_dec8"; + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "d DECIMAL(4, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "1.5") + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-9.9") + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4894,27 +4372,27 @@ public void testStringToDecimal8() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-9.9\t1970-01-01T00:00:02.000000000Z\n", + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal16() throws Exception { - String table = "test_qwp_string_to_dec16"; + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "12.5") + .longColumn("d", Long.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-99.9") + .longColumn("d", -1_000_000_000_000L) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4922,14 +4400,14 @@ public void testStringToDecimal16() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "12.5\t1970-01-01T00:00:01.000000000Z\n" + - "-99.9\t1970-01-01T00:00:02.000000000Z\n", + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal32() throws Exception { - String table = "test_qwp_string_to_dec32"; + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(6, 2), " + @@ -4939,10 +4417,10 @@ public void testStringToDecimal32() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "1234.56") + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-999.99") + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4950,201 +4428,187 @@ public void testStringToDecimal32() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "1234.56\t1970-01-01T00:00:01.000000000Z\n" + - "-999.99\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToTimestampNs() throws Exception { - String table = "test_qwp_string_to_timestamp_ns"; + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP_NS, " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("ts_col", "2022-02-25T00:00:00.000000Z") + .longColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, ts FROM " + table); + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToString() throws Exception { - String table = "test_qwp_uuid_to_string"; + public void testLongToDouble() throws Exception { + String table = "test_qwp_long_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToVarchar() throws Exception { - String table = "test_qwp_uuid_to_varchar"; + public void testLongToFloat() throws Exception { + String table = "test_qwp_long_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .longColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("f", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } - // === SYMBOL negative coercion tests === - @Test - public void testSymbolToBooleanCoercionError() throws Exception { - String table = "test_qwp_symbol_to_boolean_error"; + public void testLongToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("BOOLEAN") - ); - } - } - @Test - public void testSymbolToByteCoercionError() throws Exception { - String table = "test_qwp_symbol_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("BYTE") + "Expected coercion error mentioning LONG but got: " + msg, + msg.contains("type coercion from LONG to") && msg.contains("is not supported") ); } } @Test - public void testSymbolToCharCoercionError() throws Exception { - String table = "test_qwp_symbol_to_char_error"; + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("CHAR") - ); - } - } - @Test - public void testSymbolToDateCoercionError() throws Exception { - String table = "test_qwp_symbol_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Value in INT range should succeed sender.table(table) - .symbol("v", "hello") + .longColumn("i", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("i", -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DATE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToDecimalCoercionError() throws Exception { - String table = "test_qwp_symbol_to_decimal_error"; + public void testLongToIntOverflowError() throws Exception { + String table = "test_qwp_long_to_int_overflow"; useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("i", (long) Integer.MAX_VALUE + 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DECIMAL") + "Expected overflow error but got: " + msg, + msg.contains("integer value 2147483648 out of range for INT") ); } } @Test - public void testSymbolToDoubleCoercionError() throws Exception { - String table = "test_qwp_symbol_to_double_error"; + public void testLongToLong256CoercionError() throws Exception { + String table = "test_qwp_long_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -5152,125 +4616,157 @@ public void testSymbolToDoubleCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DOUBLE") + msg.contains("type coercion from LONG to LONG256 is not supported") ); } } @Test - public void testSymbolToFloatCoercionError() throws Exception { - String table = "test_qwp_symbol_to_float_error"; + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed sender.table(table) - .symbol("v", "hello") + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("FLOAT") - ); } + + assertTableSizeEventually(table, 2); } @Test - public void testSymbolToGeoHashCoercionError() throws Exception { - String table = "test_qwp_symbol_to_geohash_error"; + public void testLongToShortOverflowError() throws Exception { + String table = "test_qwp_long_to_short_overflow"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("GEOHASH") + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") ); } } @Test - public void testSymbolToIntCoercionError() throws Exception { - String table = "test_qwp_symbol_to_int_error"; + public void testLongToString() throws Exception { + String table = "test_qwp_long_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("INT") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToLongCoercionError() throws Exception { - String table = "test_qwp_symbol_to_long_error"; + public void testLongToSymbol() throws Exception { + String table = "test_qwp_long_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("LONG") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToLong256CoercionError() throws Exception { - String table = "test_qwp_symbol_to_long256_error"; + public void testLongToTimestamp() throws Exception { + String table = "test_qwp_long_to_timestamp"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("t", 1_000_000L) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("t", 0L) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("LONG256") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToShortCoercionError() throws Exception { - String table = "test_qwp_symbol_to_short_error"; + public void testLongToUuidCoercionError() throws Exception { + String table = "test_qwp_long_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -5278,85 +4774,91 @@ public void testSymbolToShortCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("SHORT") + msg.contains("type coercion from LONG to UUID is not supported") ); } } @Test - public void testSymbolToTimestampCoercionError() throws Exception { - String table = "test_qwp_symbol_to_timestamp_error"; + public void testLongToVarchar() throws Exception { + String table = "test_qwp_long_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("v", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_symbol_to_timestamp_ns_error"; + public void testMultipleRowsAndBatching() throws Exception { + String table = "test_qwp_multiple_rows"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + + int rowCount = 1000; try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); + for (int i = 0; i < rowCount; i++) { + sender.table(table) + .symbol("sym", "s" + (i % 10)) + .longColumn("val", i) + .doubleColumn("dbl", i * 1.5) + .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); + } sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, rowCount); } @Test - public void testSymbolToUuidCoercionError() throws Exception { - String table = "test_qwp_symbol_to_uuid_error"; + public void testNullStringToBoolean() throws Exception { + String table = "test_qwp_null_string_to_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (b BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .stringColumn("b", "true") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", null) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("UUID") - ); } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } - // === Null coercion tests === - @Test - public void testNullStringToBoolean() throws Exception { - String table = "test_qwp_null_string_to_boolean"; + public void testNullStringToByte() throws Exception { + String table = "test_qwp_null_string_to_byte"; useTable(table); - execute("CREATE TABLE " + table + " (b BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (b BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "true") + .stringColumn("b", "42") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) .stringColumn("b", null) @@ -5366,8 +4868,8 @@ public void testNullStringToBoolean() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "b\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -5440,6 +4942,29 @@ public void testNullStringToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testNullStringToFloat() throws Exception { + String table = "test_qwp_null_string_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (f FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("f", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testNullStringToGeoHash() throws Exception { String table = "test_qwp_null_string_to_geohash"; @@ -5518,6 +5043,29 @@ public void testNullStringToNumeric() throws Exception { "SELECT i, l, d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testNullStringToShort() throws Exception { + String table = "test_qwp_null_string_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (s SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testNullStringToSymbol() throws Exception { String table = "test_qwp_null_string_to_symbol"; @@ -5610,6 +5158,29 @@ public void testNullStringToUuid() throws Exception { "SELECT u, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testNullStringToVarchar() throws Exception { + String table = "test_qwp_null_string_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testNullSymbolToString() throws Exception { String table = "test_qwp_null_symbol_to_string"; @@ -5634,419 +5205,527 @@ public void testNullSymbolToString() throws Exception { } @Test - public void testNullSymbolToVarchar() throws Exception { - String table = "test_qwp_null_symbol_to_varchar"; + public void testNullSymbolToSymbol() throws Exception { + String table = "test_qwp_null_symbol_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .symbol("s", "alpha") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .symbol("v", null) + .symbol("s", null) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + + "s\tts\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + "null\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "SELECT s, ts FROM " + table + " ORDER BY ts"); } - // === BOOLEAN negative tests === - @Test - public void testBooleanToByteCoercionError() throws Exception { - String table = "test_qwp_boolean_to_byte_error"; + public void testNullSymbolToVarchar() throws Exception { + String table = "test_qwp_null_symbol_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", null) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("BYTE") - ); } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToShortCoercionError() throws Exception { - String table = "test_qwp_boolean_to_short_error"; + public void testShort() throws Exception { + String table = "test_qwp_short"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + // Short.MIN_VALUE is the null sentinel for SHORT sender.table(table) - .boolColumn("v", true) + .shortColumn("s", Short.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("SHORT") - ); } + + assertTableSizeEventually(table, 3); } @Test - public void testBooleanToIntCoercionError() throws Exception { - String table = "test_qwp_boolean_to_int_error"; + public void testShortToBooleanCoercionError() throws Exception { + String table = "test_qwp_short_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("b", (short) 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("INT") + "Expected error mentioning SHORT and BOOLEAN but got: " + msg, + msg.contains("SHORT") && msg.contains("BOOLEAN") ); } } @Test - public void testBooleanToLongCoercionError() throws Exception { - String table = "test_qwp_boolean_to_long_error"; + public void testShortToByte() throws Exception { + String table = "test_qwp_short_to_byte"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("b", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("LONG") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToFloatCoercionError() throws Exception { - String table = "test_qwp_boolean_to_float_error"; + public void testShortToByteOverflowError() throws Exception { + String table = "test_qwp_short_to_byte_overflow"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("b", (short) 128) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("FLOAT") + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") ); } } @Test - public void testBooleanToDoubleCoercionError() throws Exception { - String table = "test_qwp_boolean_to_double_error"; + public void testShortToCharCoercionError() throws Exception { + String table = "test_qwp_short_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("c", (short) 65) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DOUBLE") + "Expected error mentioning SHORT and CHAR but got: " + msg, + msg.contains("SHORT") && msg.contains("CHAR") ); } } @Test - public void testBooleanToDateCoercionError() throws Exception { - String table = "test_qwp_boolean_to_date_error"; + public void testShortToDate() throws Exception { + String table = "test_qwp_short_to_date"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + // 1000 millis = 1 second sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DATE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToUuidCoercionError() throws Exception { - String table = "test_qwp_boolean_to_uuid_error"; + public void testShortToDecimal128() throws Exception { + String table = "test_qwp_short_to_decimal128"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", Short.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", Short.MIN_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("UUID") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "32767.00\t1970-01-01T00:00:01.000000000Z\n" + + "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToLong256CoercionError() throws Exception { - String table = "test_qwp_boolean_to_long256_error"; + public void testShortToDecimal16() throws Exception { + String table = "test_qwp_short_to_decimal16"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("LONG256") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToGeoHashCoercionError() throws Exception { - String table = "test_qwp_boolean_to_geohash_error"; + public void testShortToDecimal256() throws Exception { + String table = "test_qwp_short_to_decimal256"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("GEOHASH") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToTimestampCoercionError() throws Exception { - String table = "test_qwp_boolean_to_timestamp_error"; + public void testShortToDecimal32() throws Exception { + String table = "test_qwp_short_to_decimal32"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_boolean_to_timestamp_ns_error"; + public void testShortToDecimal64() throws Exception { + String table = "test_qwp_short_to_decimal64"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToCharCoercionError() throws Exception { - String table = "test_qwp_boolean_to_char_error"; + public void testShortToDecimal8() throws Exception { + String table = "test_qwp_short_to_decimal8"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToSymbolCoercionError() throws Exception { - String table = "test_qwp_boolean_to_symbol_error"; + public void testShortToDouble() throws Exception { + String table = "test_qwp_short_to_double"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("SYMBOL") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToDecimalCoercionError() throws Exception { - String table = "test_qwp_boolean_to_decimal_error"; + public void testShortToFloat() throws Exception { + String table = "test_qwp_short_to_float"; useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("f", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("f", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DECIMAL") - ); } - } - // === FLOAT negative tests === + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } @Test - public void testFloatToBooleanCoercionError() throws Exception { - String table = "test_qwp_float_to_boolean_error"; + public void testShortToGeoHashCoercionError() throws Exception { + String table = "test_qwp_short_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("g", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write FLOAT") && msg.contains("BOOLEAN") + "Expected coercion error mentioning SHORT but got: " + msg, + msg.contains("type coercion from SHORT to") && msg.contains("is not supported") ); } } @Test - public void testFloatToCharCoercionError() throws Exception { - String table = "test_qwp_float_to_char_error"; + public void testShortToInt() throws Exception { + String table = "test_qwp_short_to_int"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("i", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("i", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write FLOAT") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDateCoercionError() throws Exception { - String table = "test_qwp_float_to_date_error"; + public void testShortToLong() throws Exception { + String table = "test_qwp_short_to_long"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("l", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("l", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToGeoHashCoercionError() throws Exception { - String table = "test_qwp_float_to_geohash_error"; + public void testShortToLong256CoercionError() throws Exception { + String table = "test_qwp_short_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6054,85 +5733,116 @@ public void testFloatToGeoHashCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + msg.contains("type coercion from SHORT to LONG256 is not supported") ); } } @Test - public void testFloatToUuidCoercionError() throws Exception { - String table = "test_qwp_float_to_uuid_error"; + public void testShortToString() throws Exception { + String table = "test_qwp_short_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("s", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } - @Test - public void testFloatToLong256CoercionError() throws Exception { - String table = "test_qwp_float_to_long256_error"; + @Test + public void testShortToSymbol() throws Exception { + String table = "test_qwp_short_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("s", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); } - } - // === DOUBLE negative tests === + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } @Test - public void testDoubleToBooleanCoercionError() throws Exception { - String table = "test_qwp_double_to_boolean_error"; + public void testShortToTimestamp() throws Exception { + String table = "test_qwp_short_to_timestamp"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .shortColumn("t", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("t", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE") && msg.contains("BOOLEAN") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToCharCoercionError() throws Exception { - String table = "test_qwp_double_to_char_error"; + public void testShortToUuidCoercionError() throws Exception { + String table = "test_qwp_short_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .shortColumn("u", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6140,813 +5850,986 @@ public void testDoubleToCharCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE") && msg.contains("CHAR") + msg.contains("type coercion from SHORT to UUID is not supported") ); } } @Test - public void testDoubleToDateCoercionError() throws Exception { - String table = "test_qwp_double_to_date_error"; + public void testShortToVarchar() throws Exception { + String table = "test_qwp_short_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("v", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("v", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToGeoHashCoercionError() throws Exception { - String table = "test_qwp_double_to_geohash_error"; + public void testString() throws Exception { + String table = "test_qwp_string"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .stringColumn("s", "hello world") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "non-ascii äöü") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "s\ttimestamp\n" + + "hello world\t1970-01-01T00:00:01.000000000Z\n" + + "non-ascii äöü\t1970-01-01T00:00:02.000000000Z\n" + + "\t1970-01-01T00:00:03.000000000Z\n" + + "null\t1970-01-01T00:00:04.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testDoubleToUuidCoercionError() throws Exception { - String table = "test_qwp_double_to_uuid_error"; + public void testStringToBoolean() throws Exception { + String table = "test_qwp_string_to_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .stringColumn("b", "true") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "false") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "1") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "0") + .at(4_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "TRUE") + .at(5_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 5); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n" + + "true\t1970-01-01T00:00:03.000000000Z\n" + + "false\t1970-01-01T00:00:04.000000000Z\n" + + "true\t1970-01-01T00:00:05.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToLong256CoercionError() throws Exception { - String table = "test_qwp_double_to_long256_error"; + public void testStringToBooleanParseError() throws Exception { + String table = "test_qwp_string_to_boolean_err"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .stringColumn("b", "yes") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse boolean from string") ); } } - // ==================== CHAR negative tests ==================== - @Test - public void testCharToBooleanCoercionError() throws Exception { - String table = "test_qwp_char_to_boolean_error"; + public void testStringToByte() throws Exception { + String table = "test_qwp_string_to_byte"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("b", "42") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "-128") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "127") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write") && msg.contains("BOOLEAN") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToSymbolCoercionError() throws Exception { - String table = "test_qwp_char_to_symbol_error"; + public void testStringToByteParseError() throws Exception { + String table = "test_qwp_string_to_byte_err"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("b", "abc") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write") && msg.contains("SYMBOL") + "Expected parse error but got: " + msg, + msg.contains("cannot parse BYTE from string") ); } } @Test - public void testCharToByteCoercionError() throws Exception { - String table = "test_qwp_char_to_byte_error"; + public void testStringToChar() throws Exception { + String table = "test_qwp_string_to_char"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("c", "A") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("c", "Hello") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("BYTE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "c\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "H\t1970-01-01T00:00:02.000000000Z\n", + "SELECT c, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToShortCoercionError() throws Exception { - String table = "test_qwp_char_to_short_error"; + public void testStringToDate() throws Exception { + String table = "test_qwp_string_to_date"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "2022-02-25T00:00:00.000Z") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("SHORT") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "d\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToIntCoercionError() throws Exception { - String table = "test_qwp_char_to_int_error"; + public void testStringToDateParseError() throws Exception { + String table = "test_qwp_string_to_date_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("v", "not_a_date") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("INT") + "Expected parse error but got: " + msg, + msg.contains("cannot parse DATE from string") && msg.contains("not_a_date") ); } } @Test - public void testCharToLongCoercionError() throws Exception { - String table = "test_qwp_char_to_long_error"; + public void testStringToDecimal128() throws Exception { + String table = "test_qwp_string_to_dec128"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("LONG") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToFloatCoercionError() throws Exception { - String table = "test_qwp_char_to_float_error"; + public void testStringToDecimal16() throws Exception { + String table = "test_qwp_string_to_dec16"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "12.5") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.9") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("FLOAT") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "12.5\t1970-01-01T00:00:01.000000000Z\n" + + "-99.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToDoubleCoercionError() throws Exception { - String table = "test_qwp_char_to_double_error"; + public void testStringToDecimal256() throws Exception { + String table = "test_qwp_string_to_dec256"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("DOUBLE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToDateCoercionError() throws Exception { - String table = "test_qwp_char_to_date_error"; + public void testStringToDecimal32() throws Exception { + String table = "test_qwp_string_to_dec32"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "1234.56") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("DATE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1234.56\t1970-01-01T00:00:01.000000000Z\n" + + "-999.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToUuidCoercionError() throws Exception { - String table = "test_qwp_char_to_uuid_error"; + public void testStringToDecimal64() throws Exception { + String table = "test_qwp_string_to_dec64"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("UUID") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToLong256CoercionError() throws Exception { - String table = "test_qwp_char_to_long256_error"; + public void testStringToDecimal8() throws Exception { + String table = "test_qwp_string_to_dec8"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "1.5") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-9.9") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("LONG256") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-9.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToGeoHashCoercionError() throws Exception { - String table = "test_qwp_char_to_geohash_error"; + public void testStringToDouble() throws Exception { + String table = "test_qwp_string_to_double"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "3.14") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-2.718") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("GEOHASH") - ); } - } - // ==================== LONG256 negative tests ==================== + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.718\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } @Test - public void testLong256ToBooleanCoercionError() throws Exception { - String table = "test_qwp_long256_to_boolean_error"; + public void testStringToDoubleParseError() throws Exception { + String table = "test_qwp_string_to_double_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("BOOLEAN") + "Expected parse error but got: " + msg, + msg.contains("cannot parse DOUBLE from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToCharCoercionError() throws Exception { - String table = "test_qwp_long256_to_char_error"; + public void testStringToFloat() throws Exception { + String table = "test_qwp_string_to_float"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("f", "3.14") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", "-2.5") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.5\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToSymbolCoercionError() throws Exception { - String table = "test_qwp_long256_to_symbol_error"; + public void testStringToFloatParseError() throws Exception { + String table = "test_qwp_string_to_float_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("SYMBOL") + "Expected parse error but got: " + msg, + msg.contains("cannot parse FLOAT from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToByteCoercionError() throws Exception { - String table = "test_qwp_long256_to_byte_error"; + public void testStringToGeoHash() throws Exception { + String table = "test_qwp_string_to_geohash"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(5c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("g", "s24se") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("g", "u33dc") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "g\tts\n" + + "s24se\t1970-01-01T00:00:01.000000000Z\n" + + "u33dc\t1970-01-01T00:00:02.000000000Z\n", + "SELECT g, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToShortCoercionError() throws Exception { - String table = "test_qwp_long256_to_short_error"; + public void testStringToGeoHashParseError() throws Exception { + String table = "test_qwp_string_to_geohash_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "!!!") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse geohash from string") && msg.contains("!!!") ); } } @Test - public void testLong256ToIntCoercionError() throws Exception { - String table = "test_qwp_long256_to_int_error"; + public void testStringToInt() throws Exception { + String table = "test_qwp_string_to_int"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("i", "42") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "-100") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "0") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToLongCoercionError() throws Exception { - String table = "test_qwp_long256_to_long_error"; + public void testStringToIntParseError() throws Exception { + String table = "test_qwp_string_to_int_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse INT from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToFloatCoercionError() throws Exception { - String table = "test_qwp_long256_to_float_error"; + public void testStringToLong() throws Exception { + String table = "test_qwp_string_to_long"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("l", "1000000000000") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("l", "-1") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "1000000000000\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToDoubleCoercionError() throws Exception { - String table = "test_qwp_long256_to_double_error"; + public void testStringToLong256() throws Exception { + String table = "test_qwp_string_to_long256"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "l LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("l", "0x01") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "l\tts\n" + + "0x01\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToDateCoercionError() throws Exception { - String table = "test_qwp_long256_to_date_error"; + public void testStringToLong256ParseError() throws Exception { + String table = "test_qwp_string_to_long256_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_long256") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse long256 from string") && msg.contains("not_a_long256") ); } } @Test - public void testLong256ToUuidCoercionError() throws Exception { - String table = "test_qwp_long256_to_uuid_error"; + public void testStringToLongParseError() throws Exception { + String table = "test_qwp_string_to_long_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse LONG from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long256_to_geohash_error"; + public void testStringToShort() throws Exception { + String table = "test_qwp_string_to_short"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("s", "1000") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "-32768") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "32767") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } - // ==================== UUID negative tests ==================== - @Test - public void testUuidToBooleanCoercionError() throws Exception { - String table = "test_qwp_uuid_to_boolean_error"; + public void testStringToShortParseError() throws Exception { + String table = "test_qwp_string_to_short_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("BOOLEAN") + "Expected parse error but got: " + msg, + msg.contains("cannot parse SHORT from string") && msg.contains("not_a_number") ); } } @Test - public void testUuidToCharCoercionError() throws Exception { - String table = "test_qwp_uuid_to_char_error"; + public void testStringToSymbol() throws Exception { + String table = "test_qwp_string_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("s", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToSymbolCoercionError() throws Exception { - String table = "test_qwp_uuid_to_symbol_error"; + public void testStringToTimestamp() throws Exception { + String table = "test_qwp_string_to_timestamp"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("SYMBOL") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToByteCoercionError() throws Exception { - String table = "test_qwp_uuid_to_byte_error"; + public void testStringToTimestampNs() throws Exception { + String table = "test_qwp_string_to_timestamp_ns"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP_NS, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("ts_col", "2022-02-25T00:00:00.000000Z") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test - public void testUuidToIntCoercionError() throws Exception { - String table = "test_qwp_uuid_to_int_error"; + public void testStringToTimestampParseError() throws Exception { + String table = "test_qwp_string_to_timestamp_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "not_a_timestamp") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse timestamp from string") && msg.contains("not_a_timestamp") ); } } @Test - public void testUuidToLongCoercionError() throws Exception { - String table = "test_qwp_uuid_to_long_error"; + public void testStringToUuid() throws Exception { + String table = "test_qwp_string_to_uuid"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "u\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT u, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToFloatCoercionError() throws Exception { - String table = "test_qwp_uuid_to_float_error"; + public void testStringToUuidParseError() throws Exception { + String table = "test_qwp_string_to_uuid_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "not-a-uuid") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse UUID from string") && msg.contains("not-a-uuid") ); } } @Test - public void testUuidToDoubleCoercionError() throws Exception { - String table = "test_qwp_uuid_to_double_error"; + public void testStringToVarchar() throws Exception { + String table = "test_qwp_string_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToDateCoercionError() throws Exception { - String table = "test_qwp_uuid_to_date_error"; + public void testSymbol() throws Exception { + String table = "test_qwp_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .symbol("s", "alpha") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "beta") + .at(2_000_000, ChronoUnit.MICROS); + // repeated value reuses dictionary entry + sender.table(table) + .symbol("s", "alpha") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\ttimestamp\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "beta\t1970-01-01T00:00:02.000000000Z\n" + + "alpha\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testUuidToLong256CoercionError() throws Exception { - String table = "test_qwp_uuid_to_long256_error"; + public void testSymbolToBooleanCoercionError() throws Exception { + String table = "test_qwp_symbol_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6954,21 +6837,20 @@ public void testUuidToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + msg.contains("cannot write SYMBOL") && msg.contains("BOOLEAN") ); } } @Test - public void testUuidToGeoHashCoercionError() throws Exception { - String table = "test_qwp_uuid_to_geohash_error"; + public void testSymbolToByteCoercionError() throws Exception { + String table = "test_qwp_symbol_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6976,22 +6858,20 @@ public void testUuidToGeoHashCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + msg.contains("cannot write SYMBOL") && msg.contains("BYTE") ); } } - // === TIMESTAMP negative coercion tests === - @Test - public void testTimestampToBooleanCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_boolean_error"; + public void testSymbolToCharCoercionError() throws Exception { + String table = "test_qwp_symbol_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6999,20 +6879,20 @@ public void testTimestampToBooleanCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("BOOLEAN") + msg.contains("cannot write SYMBOL") && msg.contains("CHAR") ); } } @Test - public void testTimestampToByteCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_byte_error"; + public void testSymbolToDateCoercionError() throws Exception { + String table = "test_qwp_symbol_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7020,20 +6900,20 @@ public void testTimestampToByteCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("BYTE") + msg.contains("cannot write SYMBOL") && msg.contains("DATE") ); } } @Test - public void testTimestampToShortCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_short_error"; + public void testSymbolToDecimalCoercionError() throws Exception { + String table = "test_qwp_symbol_to_decimal_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7041,20 +6921,20 @@ public void testTimestampToShortCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("SHORT") + msg.contains("cannot write SYMBOL") && msg.contains("DECIMAL") ); } } @Test - public void testTimestampToIntCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_int_error"; + public void testSymbolToDoubleCoercionError() throws Exception { + String table = "test_qwp_symbol_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7062,20 +6942,20 @@ public void testTimestampToIntCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("INT") + msg.contains("cannot write SYMBOL") && msg.contains("DOUBLE") ); } } @Test - public void testTimestampToLongCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_long_error"; + public void testSymbolToFloatCoercionError() throws Exception { + String table = "test_qwp_symbol_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7083,20 +6963,20 @@ public void testTimestampToLongCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("LONG") + msg.contains("cannot write SYMBOL") && msg.contains("FLOAT") ); } } @Test - public void testTimestampToFloatCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_float_error"; + public void testSymbolToGeoHashCoercionError() throws Exception { + String table = "test_qwp_symbol_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7104,20 +6984,20 @@ public void testTimestampToFloatCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("FLOAT") + msg.contains("cannot write SYMBOL") && msg.contains("GEOHASH") ); } } @Test - public void testTimestampToDoubleCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_double_error"; + public void testSymbolToIntCoercionError() throws Exception { + String table = "test_qwp_symbol_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7125,20 +7005,20 @@ public void testTimestampToDoubleCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DOUBLE") + msg.contains("cannot write SYMBOL") && msg.contains("INT") ); } } @Test - public void testTimestampToDateCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_date_error"; + public void testSymbolToLong256CoercionError() throws Exception { + String table = "test_qwp_symbol_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7146,20 +7026,20 @@ public void testTimestampToDateCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DATE") + msg.contains("cannot write SYMBOL") && msg.contains("LONG256") ); } } @Test - public void testTimestampToUuidCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_uuid_error"; + public void testSymbolToLongCoercionError() throws Exception { + String table = "test_qwp_symbol_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7167,20 +7047,20 @@ public void testTimestampToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("UUID") + msg.contains("cannot write SYMBOL") && msg.contains("LONG") ); } } @Test - public void testTimestampToLong256CoercionError() throws Exception { - String table = "test_qwp_timestamp_to_long256_error"; + public void testSymbolToShortCoercionError() throws Exception { + String table = "test_qwp_symbol_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7188,41 +7068,48 @@ public void testTimestampToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("LONG256") + msg.contains("cannot write SYMBOL") && msg.contains("SHORT") ); } } @Test - public void testTimestampToGeoHashCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_geohash_error"; + public void testSymbolToString() throws Exception { + String table = "test_qwp_symbol_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("s", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("GEOHASH") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampToCharCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_char_error"; + public void testSymbolToTimestampCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_error"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7230,20 +7117,20 @@ public void testTimestampToCharCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("CHAR") + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") ); } } @Test - public void testTimestampToSymbolCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_symbol_error"; + public void testSymbolToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_ns_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7251,20 +7138,20 @@ public void testTimestampToSymbolCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("SYMBOL") + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") ); } } @Test - public void testTimestampToDecimalCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_decimal_error"; + public void testSymbolToUuidCoercionError() throws Exception { + String table = "test_qwp_symbol_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7272,106 +7159,136 @@ public void testTimestampToDecimalCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DECIMAL") + msg.contains("cannot write SYMBOL") && msg.contains("UUID") ); } } - // === DECIMAL negative coercion tests === - @Test - public void testDecimalToBooleanCoercionError() throws Exception { - String table = "test_qwp_decimal_to_boolean_error"; + public void testSymbolToVarchar() throws Exception { + String table = "test_qwp_symbol_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("BOOLEAN") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalToByteCoercionError() throws Exception { - String table = "test_qwp_decimal_to_byte_error"; + public void testTimestampMicros() throws Exception { + String table = "test_qwp_timestamp_micros"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("BYTE") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\ttimestamp\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, timestamp FROM " + table); } @Test - public void testDecimalToShortCoercionError() throws Exception { - String table = "test_qwp_decimal_to_short_error"; + public void testTimestampMicrosToNanos() throws Exception { + String table = "test_qwp_timestamp_micros_to_nanos"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP_NS, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_111_111L; // 2022-02-25T00:00:00Z sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("SHORT") - ); } + + assertTableSizeEventually(table, 1); + // Microseconds scaled to nanoseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test - public void testDecimalToIntCoercionError() throws Exception { - String table = "test_qwp_decimal_to_int_error"; + public void testTimestampNanos() throws Exception { + String table = "test_qwp_timestamp_nanos"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_000_000_000L; // 2022-02-25T00:00:00Z in nanos + sender.table(table) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .at(tsNanos, ChronoUnit.NANOS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testTimestampNanosToMicros() throws Exception { + String table = "test_qwp_timestamp_nanos_to_micros"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_123_456_789L; sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("INT") - ); } + + assertTableSizeEventually(table, 1); + // Nanoseconds truncated to microseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test - public void testDecimalToLongCoercionError() throws Exception { - String table = "test_qwp_decimal_to_long_error"; + public void testTimestampToBooleanCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7379,20 +7296,20 @@ public void testDecimalToLongCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("LONG") + msg.contains("cannot write TIMESTAMP") && msg.contains("BOOLEAN") ); } } @Test - public void testDecimalToFloatCoercionError() throws Exception { - String table = "test_qwp_decimal_to_float_error"; + public void testTimestampToByteCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7400,20 +7317,20 @@ public void testDecimalToFloatCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("FLOAT") + msg.contains("cannot write TIMESTAMP") && msg.contains("BYTE") ); } } @Test - public void testDecimalToDoubleCoercionError() throws Exception { - String table = "test_qwp_decimal_to_double_error"; + public void testTimestampToCharCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7421,20 +7338,20 @@ public void testDecimalToDoubleCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("DOUBLE") + msg.contains("cannot write TIMESTAMP") && msg.contains("CHAR") ); } } @Test - public void testDecimalToDateCoercionError() throws Exception { - String table = "test_qwp_decimal_to_date_error"; + public void testTimestampToDateCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_date_error"; useTable(table); execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7442,20 +7359,20 @@ public void testDecimalToDateCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("DATE") + msg.contains("cannot write TIMESTAMP") && msg.contains("DATE") ); } } @Test - public void testDecimalToUuidCoercionError() throws Exception { - String table = "test_qwp_decimal_to_uuid_error"; + public void testTimestampToDecimalCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_decimal_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7463,20 +7380,20 @@ public void testDecimalToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("UUID") + msg.contains("cannot write TIMESTAMP") && msg.contains("DECIMAL") ); } } @Test - public void testDecimalToLong256CoercionError() throws Exception { - String table = "test_qwp_decimal_to_long256_error"; + public void testTimestampToDoubleCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7484,20 +7401,20 @@ public void testDecimalToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("LONG256") + msg.contains("cannot write TIMESTAMP") && msg.contains("DOUBLE") ); } } @Test - public void testDecimalToGeoHashCoercionError() throws Exception { - String table = "test_qwp_decimal_to_geohash_error"; + public void testTimestampToFloatCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7505,20 +7422,20 @@ public void testDecimalToGeoHashCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("GEOHASH") + msg.contains("cannot write TIMESTAMP") && msg.contains("FLOAT") ); } } @Test - public void testDecimalToTimestampCoercionError() throws Exception { - String table = "test_qwp_decimal_to_timestamp_error"; + public void testTimestampToGeoHashCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7526,20 +7443,20 @@ public void testDecimalToTimestampCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + msg.contains("cannot write TIMESTAMP") && msg.contains("GEOHASH") ); } } @Test - public void testDecimalToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_decimal_to_timestamp_ns_error"; + public void testTimestampToIntCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7547,20 +7464,20 @@ public void testDecimalToTimestampNsCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + msg.contains("cannot write TIMESTAMP") && msg.contains("INT") ); } } @Test - public void testDecimalToCharCoercionError() throws Exception { - String table = "test_qwp_decimal_to_char_error"; + public void testTimestampToLong256CoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7568,20 +7485,20 @@ public void testDecimalToCharCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("CHAR") + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG256") ); } } @Test - public void testDecimalToSymbolCoercionError() throws Exception { - String table = "test_qwp_decimal_to_symbol_error"; + public void testTimestampToLongCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7589,22 +7506,20 @@ public void testDecimalToSymbolCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("SYMBOL") + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG") ); } } - // === DOUBLE_ARRAY negative coercion tests === - @Test - public void testDoubleArrayToIntCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_int_error"; + public void testTimestampToShortCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7612,41 +7527,45 @@ public void testDoubleArrayToIntCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("INT") + msg.contains("cannot write TIMESTAMP") && msg.contains("SHORT") ); } } @Test - public void testDoubleArrayToStringCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_string_error"; + public void testTimestampToString() throws Exception { + String table = "test_qwp_timestamp_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("s", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("STRING") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleArrayToSymbolCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_symbol_error"; + public void testTimestampToSymbolCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_symbol_error"; useTable(table); execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7654,20 +7573,20 @@ public void testDoubleArrayToSymbolCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("SYMBOL") + msg.contains("cannot write TIMESTAMP") && msg.contains("SYMBOL") ); } } @Test - public void testDoubleArrayToTimestampCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_timestamp_error"; + public void testTimestampToUuidCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7675,366 +7594,411 @@ public void testDoubleArrayToTimestampCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("TIMESTAMP") + msg.contains("cannot write TIMESTAMP") && msg.contains("UUID") ); } } - // ==================== Additional null coercion tests ==================== - @Test - public void testNullStringToVarchar() throws Exception { - String table = "test_qwp_null_string_to_varchar"; + public void testTimestampToVarchar() throws Exception { + String table = "test_qwp_timestamp_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .stringColumn("v", "hello") + .timestampColumn("v", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("v", null) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + + assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testNullSymbolToSymbol() throws Exception { - String table = "test_qwp_null_symbol_to_symbol"; + public void testUuid() throws Exception { + String table = "test_qwp_uuid"; useTable(table); - execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("s", "alpha") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "alpha\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - @Test - public void testNullStringToByte() throws Exception { - String table = "test_qwp_null_string_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (b BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + UUID uuid1 = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "42") + .uuidColumn("u", uuid1.getLeastSignificantBits(), uuid1.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", null) + .uuidColumn("u", uuid2.getLeastSignificantBits(), uuid2.getMostSignificantBits()) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "u\ttimestamp\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + + "11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z\n", + "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testNullStringToShort() throws Exception { - String table = "test_qwp_null_string_to_short"; + public void testUuidToBooleanCoercionError() throws Exception { + String table = "test_qwp_uuid_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (s SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("s", "42") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", null) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("BOOLEAN") + ); + } } @Test - public void testNullStringToFloat() throws Exception { - String table = "test_qwp_null_string_to_float"; + public void testUuidToByteCoercionError() throws Exception { + String table = "test_qwp_uuid_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (f FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("f", "3.14") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("f", null) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } - // ==================== Additional positive coercion test ==================== - @Test - public void testStringToVarchar() throws Exception { - String table = "test_qwp_string_to_varchar"; + public void testUuidToCharCoercionError() throws Exception { + String table = "test_qwp_uuid_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "hello") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("v", "world") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("CHAR") + ); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } - // ==================== Additional parse error tests ==================== - @Test - public void testStringToIntParseError() throws Exception { - String table = "test_qwp_string_to_int_parse_error"; + public void testUuidToDateCoercionError() throws Exception { + String table = "test_qwp_uuid_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse INT from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToLongParseError() throws Exception { - String table = "test_qwp_string_to_long_parse_error"; + public void testUuidToDoubleCoercionError() throws Exception { + String table = "test_qwp_uuid_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse LONG from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToShortParseError() throws Exception { - String table = "test_qwp_string_to_short_parse_error"; + public void testUuidToFloatCoercionError() throws Exception { + String table = "test_qwp_uuid_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse SHORT from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToFloatParseError() throws Exception { - String table = "test_qwp_string_to_float_parse_error"; + public void testUuidToGeoHashCoercionError() throws Exception { + String table = "test_qwp_uuid_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse FLOAT from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToDoubleParseError() throws Exception { - String table = "test_qwp_string_to_double_parse_error"; + public void testUuidToIntCoercionError() throws Exception { + String table = "test_qwp_uuid_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse DOUBLE from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToDateParseError() throws Exception { - String table = "test_qwp_string_to_date_parse_error"; + public void testUuidToLong256CoercionError() throws Exception { + String table = "test_qwp_uuid_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_date") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse DATE from string") && msg.contains("not_a_date") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToTimestampParseError() throws Exception { - String table = "test_qwp_string_to_timestamp_parse_error"; + public void testUuidToLongCoercionError() throws Exception { + String table = "test_qwp_uuid_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_timestamp") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse timestamp from string") && msg.contains("not_a_timestamp") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToUuidParseError() throws Exception { - String table = "test_qwp_string_to_uuid_parse_error"; + public void testUuidToShortCoercionError() throws Exception { + String table = "test_qwp_uuid_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not-a-uuid") + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse UUID from string") && msg.contains("not-a-uuid") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to SHORT is not supported") ); } } @Test - public void testStringToLong256ParseError() throws Exception { - String table = "test_qwp_string_to_long256_parse_error"; + public void testUuidToString() throws Exception { + String table = "test_qwp_uuid_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("v", "not_a_long256") + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse long256 from string") && msg.contains("not_a_long256") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); } @Test - public void testStringToGeoHashParseError() throws Exception { - String table = "test_qwp_string_to_geohash_parse_error"; + public void testUuidToSymbolCoercionError() throws Exception { + String table = "test_qwp_uuid_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "!!!") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse geohash from string") && msg.contains("!!!") + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("SYMBOL") ); } } - // === Helper Methods === + @Test + public void testUuidToVarchar() throws Exception { + String table = "test_qwp_uuid_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); + } + + @Test + public void testWriteAllTypesInOneRow() throws Exception { + String table = "test_qwp_all_types"; + useTable(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + double[] arr1d = {1.0, 2.0, 3.0}; + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("sym", "test_symbol") + .boolColumn("bool_col", true) + .shortColumn("short_col", (short) 42) + .intColumn("int_col", 100_000) + .longColumn("long_col", 1_000_000_000L) + .floatColumn("float_col", 2.5f) + .doubleColumn("double_col", 3.14) + .stringColumn("string_col", "hello") + .charColumn("char_col", 'Z') + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .uuidColumn("uuid_col", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .long256Column("long256_col", 1, 0, 0, 0) + .doubleArray("arr_col", arr1d) + .decimalColumn("decimal_col", "99.99") + .at(tsMicros, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } private QwpWebSocketSender createQwpSender() { return QwpWebSocketSender.connect(getQuestDbHost(), getHttpPort()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java index 607b442..47fe527 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java @@ -53,13 +53,19 @@ public class WebSocketChannelTest extends AbstractTest { @Test - public void testBinaryRoundTripSmallPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); - } - - @Test - public void testBinaryRoundTripMediumPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096)); + public void testBinaryRoundTripAllByteValues() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int len = 256; + long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i); + } + assertBinaryRoundTrip(sendPtr, len); + } finally { + Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT); + } + }); } @Test @@ -74,19 +80,8 @@ public void testBinaryRoundTripLargePayload() throws Exception { } @Test - public void testBinaryRoundTripAllByteValues() throws Exception { - TestUtils.assertMemoryLeak(() -> { - int len = 256; - long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < len; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i); - } - assertBinaryRoundTrip(sendPtr, len); - } finally { - Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT); - } - }); + public void testBinaryRoundTripMediumPayload() throws Exception { + TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096)); } @Test @@ -135,6 +130,30 @@ public void testBinaryRoundTripRepeatedFrames() throws Exception { }); } + @Test + public void testBinaryRoundTripSmallPayload() throws Exception { + TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); + } + + /** + * Calls receiveFrame in a loop to handle the case where doReceiveFrame + * needs multiple reads to assemble a complete frame (e.g. header and + * payload arrive in separate TCP segments). + */ + private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + int remaining = (int) (deadline - System.currentTimeMillis()); + if (remaining <= 0) { + break; + } + if (channel.receiveFrame(handler, remaining)) { + return true; + } + } + return false; + } + private void assertBinaryRoundTrip(int payloadLen) throws Exception { long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); try { @@ -182,40 +201,6 @@ private void assertBinaryRoundTrip(long sendPtr, int payloadLen) throws Exceptio } } - /** - * Calls receiveFrame in a loop to handle the case where doReceiveFrame - * needs multiple reads to assemble a complete frame (e.g. header and - * payload arrive in separate TCP segments). - */ - private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) { - long deadline = System.currentTimeMillis() + timeoutMs; - while (System.currentTimeMillis() < deadline) { - int remaining = (int) (deadline - System.currentTimeMillis()); - if (remaining <= 0) { - break; - } - if (channel.receiveFrame(handler, remaining)) { - return true; - } - } - return false; - } - - private static class ReceivedPayload implements WebSocketChannel.ResponseHandler { - long ptr; - int length; - - @Override - public void onBinaryMessage(long payload, int length) { - this.ptr = payload; - this.length = length; - } - - @Override - public void onClose(int code, String reason) { - } - } - /** * Minimal WebSocket echo server. Accepts one connection, completes the * HTTP upgrade handshake, then echoes every binary frame back unmasked. @@ -224,32 +209,14 @@ public void onClose(int code, String reason) { private static class EchoServer implements AutoCloseable { private static final Pattern KEY_PATTERN = Pattern.compile("Sec-WebSocket-Key:\\s*(.+?)\\r\\n"); - - private final ServerSocket serverSocket; private final AtomicReference error = new AtomicReference<>(); + private final ServerSocket serverSocket; private Thread thread; EchoServer() throws IOException { serverSocket = new ServerSocket(0); } - int getPort() { - return serverSocket.getLocalPort(); - } - - void start() { - thread = new Thread(this::run, "ws-echo-server"); - thread.setDaemon(true); - thread.start(); - } - - void assertNoError() { - Throwable t = error.get(); - if (t != null) { - throw new AssertionError("echo server error", t); - } - } - @Override public void close() throws Exception { serverSocket.close(); @@ -258,24 +225,6 @@ public void close() throws Exception { } } - private void run() { - try (Socket client = serverSocket.accept()) { - client.setSoTimeout(10_000); - client.setTcpNoDelay(true); - InputStream in = client.getInputStream(); - OutputStream out = new BufferedOutputStream(client.getOutputStream()); - - completeHandshake(in, out); - echoFrames(in, out); - } catch (IOException e) { - if (!serverSocket.isClosed()) { - error.set(e); - } - } catch (Throwable t) { - error.set(t); - } - } - private void completeHandshake(InputStream in, OutputStream out) throws IOException { byte[] buf = new byte[4096]; int pos = 0; @@ -389,10 +338,18 @@ private void echoFrames(InputStream in, OutputStream out) throws IOException { byte m3 = readBuf[maskKeyOffset + 3]; for (int i = 0; i < (int) payloadLength; i++) { switch (i & 3) { - case 0: readBuf[headerSize + i] ^= m0; break; - case 1: readBuf[headerSize + i] ^= m1; break; - case 2: readBuf[headerSize + i] ^= m2; break; - case 3: readBuf[headerSize + i] ^= m3; break; + case 0: + readBuf[headerSize + i] ^= m0; + break; + case 1: + readBuf[headerSize + i] ^= m1; + break; + case 2: + readBuf[headerSize + i] ^= m2; + break; + case 3: + readBuf[headerSize + i] ^= m3; + break; } } } @@ -426,5 +383,55 @@ private void echoFrames(InputStream in, OutputStream out) throws IOException { out.flush(); } } + + private void run() { + try (Socket client = serverSocket.accept()) { + client.setSoTimeout(10_000); + client.setTcpNoDelay(true); + InputStream in = client.getInputStream(); + OutputStream out = new BufferedOutputStream(client.getOutputStream()); + + completeHandshake(in, out); + echoFrames(in, out); + } catch (IOException e) { + if (!serverSocket.isClosed()) { + error.set(e); + } + } catch (Throwable t) { + error.set(t); + } + } + + void assertNoError() { + Throwable t = error.get(); + if (t != null) { + throw new AssertionError("echo server error", t); + } + } + + int getPort() { + return serverSocket.getLocalPort(); + } + + void start() { + thread = new Thread(this::run, "ws-echo-server"); + thread.setDaemon(true); + thread.start(); + } + } + + private static class ReceivedPayload implements WebSocketChannel.ResponseHandler { + int length; + long ptr; + + @Override + public void onBinaryMessage(long payload, int length) { + this.ptr = payload; + this.length = length; + } + + @Override + public void onClose(int code, String reason) { + } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java index 073bf27..6034988 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -29,101 +29,29 @@ import io.questdb.client.std.Unsafe; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class OffHeapAppendMemoryTest { @Test - public void testPutAndReadByte() { + public void testCloseFreesMemory() { long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putByte((byte) 42); - mem.putByte((byte) -1); - mem.putByte((byte) 0); + OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); + long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertTrue(during > before); - assertEquals(3, mem.getAppendOffset()); - assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - } + mem.close(); long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); assertEquals(before, after); } @Test - public void testPutAndReadShort() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putShort((short) 12_345); - mem.putShort(Short.MIN_VALUE); - mem.putShort(Short.MAX_VALUE); - - assertEquals(6, mem.getAppendOffset()); - assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); - assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); - assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); - } - } - - @Test - public void testPutAndReadInt() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(100_000); - mem.putInt(Integer.MIN_VALUE); - - assertEquals(8, mem.getAppendOffset()); - assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); - } - } - - @Test - public void testPutAndReadLong() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putLong(1_000_000_000_000L); - mem.putLong(Long.MIN_VALUE); - - assertEquals(16, mem.getAppendOffset()); - assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); - assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); - } - } - - @Test - public void testPutAndReadFloat() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putFloat(3.14f); - mem.putFloat(Float.NaN); - - assertEquals(8, mem.getAppendOffset()); - assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); - assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); - } - } - - @Test - public void testPutAndReadDouble() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putDouble(2.718281828); - mem.putDouble(Double.NaN); - - assertEquals(16, mem.getAppendOffset()); - assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); - assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); - } - } - - @Test - public void testPutBoolean() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putBoolean(true); - mem.putBoolean(false); - mem.putBoolean(true); - - assertEquals(3, mem.getAppendOffset()); - assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - } + public void testDoubleCloseIsSafe() { + OffHeapAppendMemory mem = new OffHeapAppendMemory(); + mem.putInt(42); + mem.close(); + mem.close(); // should not throw } @Test @@ -144,24 +72,6 @@ public void testGrowth() { assertEquals(before, after); } - @Test - public void testTruncate() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(1); - mem.putInt(2); - mem.putInt(3); - assertEquals(12, mem.getAppendOffset()); - - mem.truncate(); - assertEquals(0, mem.getAppendOffset()); - - // Can write again after truncate - mem.putInt(42); - assertEquals(4, mem.getAppendOffset()); - assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - } - } - @Test public void testJumpTo() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { @@ -183,15 +93,41 @@ public void testJumpTo() { } @Test - public void testSkip() { + public void testLargeGrowth() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write 10000 doubles to stress growth + for (int i = 0; i < 10_000; i++) { + mem.putDouble(i * 1.1); + } + assertEquals(80_000, mem.getAppendOffset()); + + // Verify first and last values + assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testMixedTypes() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(1); - mem.skip(8); - mem.putInt(2); + mem.putByte((byte) 1); + mem.putShort((short) 2); + mem.putInt(3); + mem.putLong(4L); + mem.putFloat(5.0f); + mem.putDouble(6.0); - assertEquals(16, mem.getAppendOffset()); - assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + long addr = mem.pageAddress(); + assertEquals(1, Unsafe.getUnsafe().getByte(addr)); + assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); + assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); + assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); + assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); + assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); + assertEquals(27, mem.getAppendOffset()); } } @@ -206,62 +142,96 @@ public void testPageAddress() { } @Test - public void testCloseFreesMemory() { + public void testPutAndReadByte() { long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); - long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertTrue(during > before); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 42); + mem.putByte((byte) -1); + mem.putByte((byte) 0); - mem.close(); + assertEquals(3, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); assertEquals(before, after); } @Test - public void testDoubleCloseIsSafe() { - OffHeapAppendMemory mem = new OffHeapAppendMemory(); - mem.putInt(42); - mem.close(); - mem.close(); // should not throw + public void testPutAndReadDouble() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putDouble(2.718281828); + mem.putDouble(Double.NaN); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); + } } @Test - public void testMixedTypes() { + public void testPutAndReadFloat() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putByte((byte) 1); - mem.putShort((short) 2); - mem.putInt(3); - mem.putLong(4L); - mem.putFloat(5.0f); - mem.putDouble(6.0); + mem.putFloat(3.14f); + mem.putFloat(Float.NaN); - long addr = mem.pageAddress(); - assertEquals(1, Unsafe.getUnsafe().getByte(addr)); - assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); - assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); - assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); - assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); - assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); - assertEquals(27, mem.getAppendOffset()); + assertEquals(8, mem.getAppendOffset()); + assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); + assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); } } @Test - public void testLargeGrowth() { - long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { - // Write 10000 doubles to stress growth - for (int i = 0; i < 10_000; i++) { - mem.putDouble(i * 1.1); - } - assertEquals(80_000, mem.getAppendOffset()); + public void testPutAndReadInt() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(100_000); + mem.putInt(Integer.MIN_VALUE); - // Verify first and last values - assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); - assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); + assertEquals(8, mem.getAppendOffset()); + assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); + } + } + + @Test + public void testPutAndReadLong() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(1_000_000_000_000L); + mem.putLong(Long.MIN_VALUE); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + } + + @Test + public void testPutAndReadShort() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putShort((short) 12_345); + mem.putShort(Short.MIN_VALUE); + mem.putShort(Short.MAX_VALUE); + + assertEquals(6, mem.getAppendOffset()); + assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); + assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); + assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); + } + } + + @Test + public void testPutBoolean() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putBoolean(true); + mem.putBoolean(false); + mem.putBoolean(true); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); } - long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertEquals(before, after); } @Test @@ -288,6 +258,15 @@ public void testPutUtf8Empty() { } } + @Test + public void testPutUtf8Mixed() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes + mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00"); + assertEquals(10, mem.getAppendOffset()); + } + } + @Test public void testPutUtf8MultiByte() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { @@ -333,11 +312,33 @@ public void testPutUtf8ThreeByte() { } @Test - public void testPutUtf8Mixed() { + public void testSkip() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes - mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00"); - assertEquals(10, mem.getAppendOffset()); + mem.putInt(1); + mem.skip(8); + mem.putInt(2); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + } + } + + @Test + public void testTruncate() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.putInt(2); + mem.putInt(3); + assertEquals(12, mem.getAppendOffset()); + + mem.truncate(); + assertEquals(0, mem.getAppendOffset()); + + // Can write again after truncate + mem.putInt(42); + assertEquals(4, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java index 11f4c36..0b516e0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java @@ -36,52 +36,79 @@ public class QwpBitWriterTest { @Test - public void testWriteBitsThrowsOnOverflow() { - long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + public void testFlushThrowsOnOverflow() { + long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 4); - // Fill the buffer (32 bits = 4 bytes) - writer.writeBits(0xFFFF_FFFFL, 32); - // Next write should throw — buffer is full + writer.reset(ptr, 1); + // Write 8 bits to fill the single byte + writer.writeBits(0xFF, 8); + // Write a few more bits that sit in the bit buffer + writer.writeBits(0x3, 4); + // Flush should throw because there's no room for the partial byte try { - writer.writeBits(1, 8); + writer.flush(); + fail("expected LineSenderException on buffer overflow during flush"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() { + // Source: 1 timestamp (8 bytes), dest: only 4 bytes + long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + try { + Unsafe.getUnsafe().putLong(src, 1_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + try { + encoder.encodeTimestamps(dst, 4, src, 1); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } } finally { - Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testWriteByteThrowsOnOverflow() { - long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); + public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() { + // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second) + long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS); try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 1); - writer.writeByte(0x42); + Unsafe.getUnsafe().putLong(src, 1_000_000L); + Unsafe.getUnsafe().putLong(src + 8, 2_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); try { - writer.writeByte(0x43); + encoder.encodeTimestamps(dst, 12, src, 2); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } } finally { - Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testWriteIntThrowsOnOverflow() { + public void testWriteBitsThrowsOnOverflow() { long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); writer.reset(ptr, 4); - writer.writeInt(42); + // Fill the buffer (32 bits = 4 bytes) + writer.writeBits(0xFFFF_FFFFL, 32); + // Next write should throw — buffer is full try { - writer.writeInt(99); + writer.writeBits(1, 8); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); @@ -92,37 +119,30 @@ public void testWriteIntThrowsOnOverflow() { } @Test - public void testWriteLongThrowsOnOverflow() { + public void testWriteBitsWithinCapacitySucceeds() { long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); writer.reset(ptr, 8); - writer.writeLong(42L); - try { - writer.writeLong(99L); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); - } + writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64); + writer.flush(); + assertEquals(8, writer.getPosition() - ptr); + assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr)); } finally { Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testFlushThrowsOnOverflow() { + public void testWriteByteThrowsOnOverflow() { long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); writer.reset(ptr, 1); - // Write 8 bits to fill the single byte - writer.writeBits(0xFF, 8); - // Write a few more bits that sit in the bit buffer - writer.writeBits(0x3, 4); - // Flush should throw because there's no room for the partial byte + writer.writeByte(0x42); try { - writer.flush(); - fail("expected LineSenderException on buffer overflow during flush"); + writer.writeByte(0x43); + fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } @@ -131,59 +151,37 @@ public void testFlushThrowsOnOverflow() { } } - // --- QwpGorillaEncoder overflow tests --- - @Test - public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() { - // Source: 1 timestamp (8 bytes), dest: only 4 bytes - long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); - long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + public void testWriteIntThrowsOnOverflow() { + long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); try { - Unsafe.getUnsafe().putLong(src, 1_000_000L); - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 4); + writer.writeInt(42); try { - encoder.encodeTimestamps(dst, 4, src, 1); + writer.writeInt(99); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() { - // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second) - long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS); + public void testWriteLongThrowsOnOverflow() { + long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); try { - Unsafe.getUnsafe().putLong(src, 1_000_000L); - Unsafe.getUnsafe().putLong(src + 8, 2_000_000L); - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 8); + writer.writeLong(42L); try { - encoder.encodeTimestamps(dst, 12, src, 2); + writer.writeLong(99L); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS); - } - } - - @Test - public void testWriteBitsWithinCapacitySucceeds() { - long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 8); - writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64); - writer.flush(); - assertEquals(8, writer.getPosition() - ptr); - assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr)); } finally { Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java index 2bd28b9..2f5a3ac 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java @@ -28,7 +28,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class QwpColumnDefTest { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 501893e..e4ac0d7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -29,88 +29,11 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; public class QwpTableBufferTest { - /** - * Simulates the encoder's walk over array data — the same logic as - * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat - * double values the encoder would serialize for the given column. - */ - private static double[] readDoubleArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - double[] data = col.getDoubleArrayData(); - int count = col.getValueCount(); - - // First pass: count total elements - int totalElements = 0; - int shapeIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - totalElements += elemCount; - } - - // Second pass: collect values - double[] result = new double[totalElements]; - shapeIdx = 0; - int dataIdx = 0; - int resultIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - for (int e = 0; e < elemCount; e++) { - result[resultIdx++] = data[dataIdx++]; - } - } - return result; - } - - /** - * Same as above but for long arrays (mirrors QwpWebSocketEncoder.writeLongArrayColumn()). - */ - private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - long[] data = col.getLongArrayData(); - int count = col.getValueCount(); - - int totalElements = 0; - int shapeIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - totalElements += elemCount; - } - - long[] result = new long[totalElements]; - shapeIdx = 0; - int dataIdx = 0; - int resultIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - for (int e = 0; e < elemCount; e++) { - result[resultIdx++] = data[dataIdx++]; - } - } - return result; - } - @Test public void testCancelRowRewindsDoubleArrayOffsets() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { @@ -377,4 +300,82 @@ public void testLongArrayShrinkingSize() { assertEquals(2, shapes[1]); } } + + /** + * Simulates the encoder's walk over array data — the same logic as + * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat + * double values the encoder would serialize for the given column. + */ + private static double[] readDoubleArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + int count = col.getValueCount(); + + // First pass: count total elements + int totalElements = 0; + int shapeIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + totalElements += elemCount; + } + + // Second pass: collect values + double[] result = new double[totalElements]; + shapeIdx = 0; + int dataIdx = 0; + int resultIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + for (int e = 0; e < elemCount; e++) { + result[resultIdx++] = data[dataIdx++]; + } + } + return result; + } + + /** + * Same as above but for long arrays (mirrors QwpWebSocketEncoder.writeLongArrayColumn()). + */ + private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + int count = col.getValueCount(); + + int totalElements = 0; + int shapeIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + totalElements += elemCount; + } + + long[] result = new long[totalElements]; + shapeIdx = 0; + int dataIdx = 0; + int resultIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + for (int e = 0; e < elemCount; e++) { + result[resultIdx++] = data[dataIdx++]; + } + } + return result; + } } From 6e1395948d03cd528d8664847ce5ca54f79d87fb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:03:40 +0100 Subject: [PATCH 040/230] Fix Gorilla encoder bucket boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bucket boundary constants for two's complement signed ranges were inverted — the min and max magnitudes were swapped. For an N-bit two's complement integer the valid range is [-2^(N-1), 2^(N-1) - 1], so: 7-bit: [-64, 63] not [-63, 64] 9-bit: [-256, 255] not [-255, 256] 12-bit: [-2048, 2047] not [-2047, 2048] With the old boundaries, a value like 64 would be placed in the 7-bit bucket, but 64 in 7-bit two's complement decodes as -64, silently corrupting timestamp data at bucket boundaries. Co-Authored-By: Claude Opus 4.6 --- .../qwp/protocol/QwpGorillaEncoder.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 082f6b2..a21215a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -36,9 +36,9 @@ * DoD = (t[n] - t[n-1]) - (t[n-1] - t[n-2]) * * if DoD == 0: write '0' (1 bit) - * elif DoD in [-63, 64]: write '10' + 7-bit (9 bits) - * elif DoD in [-255, 256]: write '110' + 9-bit (12 bits) - * elif DoD in [-2047, 2048]: write '1110' + 12-bit (16 bits) + * elif DoD in [-64, 63]: write '10' + 7-bit (9 bits) + * elif DoD in [-256, 255]: write '110' + 9-bit (12 bits) + * elif DoD in [-2048, 2047]: write '1110' + 12-bit (16 bits) * else: write '1111' + 32-bit (36 bits) * *

@@ -47,13 +47,13 @@ */ public class QwpGorillaEncoder { - private static final int BUCKET_12BIT_MAX = 2048; - private static final int BUCKET_12BIT_MIN = -2047; - private static final int BUCKET_7BIT_MAX = 64; + private static final int BUCKET_12BIT_MAX = 2047; + private static final int BUCKET_12BIT_MIN = -2048; + private static final int BUCKET_7BIT_MAX = 63; // Bucket boundaries (two's complement signed ranges) - private static final int BUCKET_7BIT_MIN = -63; - private static final int BUCKET_9BIT_MAX = 256; - private static final int BUCKET_9BIT_MIN = -255; + private static final int BUCKET_7BIT_MIN = -64; + private static final int BUCKET_9BIT_MAX = 255; + private static final int BUCKET_9BIT_MIN = -256; private final QwpBitWriter bitWriter = new QwpBitWriter(); /** @@ -202,15 +202,15 @@ public void encodeDoD(long deltaOfDelta) { case 0: // DoD == 0 bitWriter.writeBit(0); break; - case 1: // [-63, 64] -> '10' + 7-bit + case 1: // [-64, 63] -> '10' + 7-bit bitWriter.writeBits(0b01, 2); bitWriter.writeSigned(deltaOfDelta, 7); break; - case 2: // [-255, 256] -> '110' + 9-bit + case 2: // [-256, 255] -> '110' + 9-bit bitWriter.writeBits(0b011, 3); bitWriter.writeSigned(deltaOfDelta, 9); break; - case 3: // [-2047, 2048] -> '1110' + 12-bit + case 3: // [-2048, 2047] -> '1110' + 12-bit bitWriter.writeBits(0b0111, 4); bitWriter.writeSigned(deltaOfDelta, 12); break; From b8514c08d6c880664e00f76ac73b602691645032 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:21:22 +0100 Subject: [PATCH 041/230] Remove unused opcode param from beginFrame() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opcode parameter in beginFrame(int) was accepted but never stored or used — the opcode only matters when writing the frame header in endFrame(int), where all callers already pass the correct value. Remove the misleading parameter to make the API honest. Also remove beginBinaryFrame() and beginTextFrame() which were just wrappers passing an unused opcode. The single caller in WebSocketClient is updated to call beginFrame() directly. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 2 +- .../http/client/WebSocketSendBuffer.java | 22 +++---------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index a0cf1a3..32d79dd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -294,7 +294,7 @@ public boolean receiveFrame(WebSocketFrameHandler handler) { public void sendBinary(long dataPtr, int length, int timeout) { checkConnected(); sendBuffer.reset(); - sendBuffer.beginBinaryFrame(); + sendBuffer.beginFrame(); sendBuffer.putBlockOfBytes(dataPtr, length); WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 861de15..287bda0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -103,19 +103,10 @@ public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) { } /** - * Begins a new binary WebSocket frame. Reserves space for the maximum header size. - * After calling this method, use ArrayBufferAppender methods to write the payload. + * Begins a new WebSocket frame. Reserves space for the maximum header size. + * The opcode is specified later when ending the frame via {@link #endFrame(int)}. */ - public void beginBinaryFrame() { - beginFrame(WebSocketOpcode.BINARY); - } - - /** - * Begins a new WebSocket frame with the specified opcode. - * - * @param opcode the frame opcode - */ - public void beginFrame(int opcode) { + public void beginFrame() { frameStartOffset = writePos; // Reserve maximum header space ensureCapacity(MAX_HEADER_SIZE); @@ -123,13 +114,6 @@ public void beginFrame(int opcode) { payloadStartOffset = writePos; } - /** - * Begins a new text WebSocket frame. Reserves space for the maximum header size. - */ - public void beginTextFrame() { - beginFrame(WebSocketOpcode.TEXT); - } - @Override public void close() { if (bufPtr != 0) { From 7a87c76704c87cc198c914817f10c3ad76303bae Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:25:57 +0100 Subject: [PATCH 042/230] Use correct default port for WebSocket protocol The single-host fallback in configuration string parsing used DEFAULT_HTTP_PORT for all non-TCP protocols. This works by coincidence since DEFAULT_HTTP_PORT and DEFAULT_WEBSOCKET_PORT are both 9000, but is semantically incorrect. Use the proper DEFAULT_WEBSOCKET_PORT constant when the protocol is WebSocket. Also clean up Javadoc: remove a dangling isRetryable() reference from Sender.java, fix grammar ("allows to use" -> "allows using"), and tidy LineSenderException Javadoc. Co-Authored-By: Claude Opus 4.6 --- core/src/main/java/io/questdb/client/Sender.java | 8 +++++--- .../client/cutlass/line/LineSenderException.java | 12 +++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index c998b78..755bd78 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -96,7 +96,7 @@ * 2. Call {@link #reset()} to clear the internal buffers and start building a new row *
* Note: If the underlying error is permanent, retrying {@link #flush()} will fail again. - * Use {@link #reset()} to discard the problematic data and continue with new data. See {@link LineSenderException#isRetryable()} + * Use {@link #reset()} to discard the problematic data and continue with new data. * */ public interface Sender extends Closeable, ArraySender { @@ -109,7 +109,7 @@ public interface Sender extends Closeable, ArraySender { /** * Create a Sender builder instance from a configuration string. *
- * This allows to use the configuration string as a template for creating a Sender builder instance and then + * This allows using the configuration string as a template for creating a Sender builder instance and then * tune options which are not available in the configuration string. Configurations options specified in the * configuration string cannot be overridden via the builder methods. *

@@ -1529,7 +1529,9 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { address(sink); if (ports.size() == hosts.size() - 1) { // not set - port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT : DEFAULT_HTTP_PORT); + port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT + : protocol == PROTOCOL_WEBSOCKET ? DEFAULT_WEBSOCKET_PORT + : DEFAULT_HTTP_PORT); } } else if (Chars.equals("user", sink)) { // deprecated key: user, new key: username diff --git a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java index 9c6cc16..6fdaf9a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java @@ -39,15 +39,9 @@ *

  • For permanent errors: Either close and recreate the Sender, or call {@code reset()} to clear * the buffer and continue with new data
    • * + *

    + * @see io.questdb.client.Sender * - *

    Retryability

    - * The {@link #isRetryable()} method provides a best-effort indication of whether the error - * might be resolved by retrying at the application level. This is particularly important - * because this exception is only thrown after the sender has exhausted its own internal - * retry attempts. The retryability flag helps applications decide whether to implement - * additional retry logic with longer delays or different strategies. - * - * @see io.questdb.client.Sender * @see io.questdb.client.Sender#flush() * @see io.questdb.client.Sender#reset() */ @@ -115,4 +109,4 @@ public LineSenderException putAsPrintable(CharSequence nonPrintable) { message.putAsPrintable(nonPrintable); return this; } -} \ No newline at end of file +} From a274be53038f9e02817a483a379379343f411bef Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:35:50 +0100 Subject: [PATCH 043/230] Fix inc() not adding key to list inc() called putAt0() directly without adding the key to the `list` field. This caused keys() to be incomplete and valueQuick() to return wrong results for keys inserted via inc(). Add the missing list.add() call, consistent with putAt() and putIfAbsent(). Co-Authored-By: Claude Opus 4.6 --- .../java/io/questdb/client/std/CharSequenceIntHashMap.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java index 932d1eb..d56c181 100644 --- a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -67,7 +67,9 @@ public void inc(@NotNull CharSequence key) { if (index < 0) { values[-index - 1]++; } else { - putAt0(index, Chars.toString(key), 1); + String keyString = Chars.toString(key); + putAt0(index, keyString, 1); + list.add(keyString); } } From 7d003e08eb9a08dc38c5630e599ef0106cc8aa2d Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 11:04:40 +0100 Subject: [PATCH 044/230] Validate low surrogate in UTF-8 encoding When a high surrogate was detected in the UTF-8 encoding paths, the next char was consumed and used as a low surrogate without validating it was actually in the [0xDC00, 0xDFFF] range. This produced garbage 4-byte sequences and silently swallowed the following character. Add Character.isLowSurrogate(c2) checks in all 5 putUtf8/hasher encoding sites and both utf8Length methods. Invalid surrogates now emit '?' and re-process the consumed char on the next iteration, consistent with Utf8s.encodeUtf16Surrogate(). Co-Authored-By: Claude Opus 4.6 --- .../http/client/WebSocketSendBuffer.java | 15 +++-- .../qwp/client/NativeBufferWriter.java | 19 ++++-- .../cutlass/qwp/client/QwpBufferWriter.java | 4 +- .../qwp/protocol/OffHeapAppendMemory.java | 15 +++-- .../cutlass/qwp/protocol/QwpSchemaHash.java | 30 ++++++--- .../http/client/WebSocketSendBufferTest.java | 46 +++++++++++++ .../qwp/client/NativeBufferWriterTest.java | 37 +++++++++++ .../qwp/protocol/OffHeapAppendMemoryTest.java | 12 ++++ .../protocol/QwpSchemaHashSurrogateTest.java | 66 +++++++++++++++++++ 9 files changed, 217 insertions(+), 27 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 287bda0..66cdf13 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -354,11 +354,16 @@ public void putUtf8(String value) { putByte((byte) (0x80 | (c & 0x3F))); } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { char c2 = value.charAt(++i); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - putByte((byte) (0xF0 | (codePoint >> 18))); - putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - putByte((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) '?'); + i--; + } } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index a11f70f..1e00e12 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -66,9 +66,11 @@ public static int utf8Length(String s) { len++; } else if (c < 0x800) { len += 2; - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { i++; len += 4; + } else if (Character.isSurrogate(c)) { + len++; } else { len += 3; } @@ -243,11 +245,16 @@ public void putUtf8(String value) { putByte((byte) (0x80 | (c & 0x3F))); } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { char c2 = value.charAt(++i); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - putByte((byte) (0xF0 | (codePoint >> 18))); - putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - putByte((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) '?'); + i--; + } } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java index c50cdf6..f32f575 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java @@ -59,9 +59,11 @@ static int utf8Length(String s) { len++; } else if (c < 0x800) { len += 2; - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { i++; len += 4; + } else if (Character.isSurrogate(c)) { + len++; } else { len += 3; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index b73d88c..e02398f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -152,11 +152,16 @@ public void putUtf8(String value) { Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F))); } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < len) { char c2 = value.charAt(++i); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18))); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F))); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F))); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F))); + } else { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); + i--; + } } else { Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12))); Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index dd6388a..b62fc07 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -112,11 +112,16 @@ public static long computeSchemaHash(String[] columnNames, byte[] columnTypes) { } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { // Surrogate pair (4 bytes) char c2 = name.charAt(++j); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - hasher.update((byte) (0xF0 | (codePoint >> 18))); - hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - hasher.update((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) '?'); + j--; + } } else { // Three bytes hasher.update((byte) (0xE0 | (c >> 12))); @@ -180,11 +185,16 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList= 0xD800 && c <= 0xDBFF && j + 1 < len) { char c2 = name.charAt(++j); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - hasher.update((byte) (0xF0 | (codePoint >> 18))); - hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - hasher.update((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) '?'); + j--; + } } else { hasher.update((byte) (0xE0 | (c >> 12))); hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java new file mode 100644 index 0000000..2218e9d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class WebSocketSendBufferTest { + + @Test + public void testPutUtf8InvalidSurrogatePair() { + try (WebSocketSendBuffer buf = new WebSocketSendBuffer(256)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + buf.putUtf8("\uD800X"); + assertEquals(2, buf.getWritePos()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(buf.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 1)); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index f51aa54..a22896d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.std.Unsafe; import org.junit.Test; @@ -74,6 +75,42 @@ public void testSkipAdvancesPosition() { } } + @Test + public void testPutUtf8InvalidSurrogatePair() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + writer.putUtf8("\uD800X"); + assertEquals(2, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + } + + @Test + public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, QwpBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, QwpBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, QwpBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, QwpBufferWriter.utf8Length("\uD83D\uDE00")); + } + @Test public void testSkipThenPatchInt() { try (NativeBufferWriter writer = new NativeBufferWriter(8)) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java index 6034988..d7755f4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -286,6 +286,18 @@ public void testPutUtf8Null() { } } + @Test + public void testPutUtf8InvalidSurrogatePair() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + mem.putUtf8("\uD800X"); + assertEquals(2, mem.getAppendOffset()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(mem.addressOf(1))); + } + } + @Test public void testPutUtf8SurrogatePairs() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java new file mode 100644 index 0000000..76610f7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java @@ -0,0 +1,66 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaHash; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.ObjList; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class QwpSchemaHashSurrogateTest { + + private static final byte TYPE_LONG = 0x05; + + @Test + public void testComputeSchemaHashInvalidSurrogatePair() { + byte[] types = {TYPE_LONG}; + + // "\uD800X" has a high surrogate followed by non-low-surrogate 'X'. + // With the fix, the high surrogate becomes '?' and 'X' is preserved, + // so the hash should equal the hash of "?X". + long hashInvalid = QwpSchemaHash.computeSchemaHash( + new String[]{"\uD800X"}, types + ); + long hashExpected = QwpSchemaHash.computeSchemaHash( + new String[]{"?X"}, types + ); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashDirectInvalidSurrogatePair() { + ObjList invalidCols = new ObjList<>(); + invalidCols.add(new QwpTableBuffer.ColumnBuffer("\uD800X", TYPE_LONG, false)); + + ObjList expectedCols = new ObjList<>(); + expectedCols.add(new QwpTableBuffer.ColumnBuffer("?X", TYPE_LONG, false)); + + long hashInvalid = QwpSchemaHash.computeSchemaHashDirect(invalidCols); + long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); + assertEquals(hashExpected, hashInvalid); + } +} From 11326f1388be6e93f4660319a7d974e3e013b99a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 12:08:28 +0100 Subject: [PATCH 045/230] Eliminate hot-path allocations in waitForAck The boolean[] wrapper and anonymous WebSocketFrameHandler were allocated on every loop iteration inside waitForAck(), generating GC pressure on the data ingestion hot path. Hoist both into reusable instance fields: ackResponse (WebSocket response buffer), sawBinaryAck (plain boolean replacing the boolean[] wrapper), and ackHandler (a static nested class AckFrameHandler replacing the per-iteration anonymous class). Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index b2aa9a3..b597d34 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -133,6 +133,8 @@ public class QwpWebSocketSender implements Sender { private final LongHashSet sentSchemaHashes = new LongHashSet(); private final CharSequenceObjHashMap tableBuffers; private final boolean tlsEnabled; + private final AckFrameHandler ackHandler = new AckFrameHandler(this); + private final WebSocketResponse ackResponse = new WebSocketResponse(); private MicrobatchBuffer activeBuffer; // Double-buffering for async I/O private MicrobatchBuffer buffer0; @@ -160,6 +162,7 @@ public class QwpWebSocketSender implements Sender { private long nextBatchSequence = 0; // Async mode: pending row tracking private int pendingRowCount; + private boolean sawBinaryAck; private WebSocketSendQueue sendQueue; private QwpWebSocketSender( @@ -1386,39 +1389,20 @@ private long toMicros(long value, ChronoUnit unit) { * Waits synchronously for an ACK from the server for the specified batch. */ private void waitForAck(long expectedSequence) { - WebSocketResponse response = new WebSocketResponse(); long deadline = System.currentTimeMillis() + InFlightWindow.DEFAULT_TIMEOUT_MS; while (System.currentTimeMillis() < deadline) { try { - final boolean[] sawBinary = {false}; - boolean received = client.receiveFrame(new WebSocketFrameHandler() { - @Override - public void onBinaryMessage(long payloadPtr, int payloadLen) { - sawBinary[0] = true; - if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { - throw new LineSenderException( - "Invalid ACK response payload [length=" + payloadLen + ']' - ); - } - if (!response.readFrom(payloadPtr, payloadLen)) { - throw new LineSenderException("Failed to parse ACK response"); - } - } - - @Override - public void onClose(int code, String reason) { - throw new LineSenderException("WebSocket closed while waiting for ACK: " + reason); - } - }, 1000); // 1 second timeout per read attempt + sawBinaryAck = false; + boolean received = client.receiveFrame(ackHandler, 1000); // 1 second timeout per read attempt if (received) { // Non-binary frames (e.g. ping/pong/text) are not ACKs. - if (!sawBinary[0]) { + if (!sawBinaryAck) { continue; } - long sequence = response.getSequence(); - if (response.isSuccess()) { + long sequence = ackResponse.getSequence(); + if (ackResponse.isSuccess()) { // Cumulative ACK - acknowledge all batches up to this sequence inFlightWindow.acknowledgeUpTo(sequence); if (sequence >= expectedSequence) { @@ -1426,10 +1410,10 @@ public void onClose(int code, String reason) { } // Got ACK for lower sequence - continue waiting } else { - String errorMessage = response.getErrorMessage(); + String errorMessage = ackResponse.getErrorMessage(); LineSenderException error = new LineSenderException( "Server error for batch " + sequence + ": " + - response.getStatusName() + " - " + errorMessage); + ackResponse.getStatusName() + " - " + errorMessage); inFlightWindow.fail(sequence, error); if (sequence == expectedSequence) { throw error; @@ -1450,4 +1434,30 @@ public void onClose(int code, String reason) { failExpectedIfNeeded(expectedSequence, timeout); throw timeout; } + + private static class AckFrameHandler implements WebSocketFrameHandler { + private final QwpWebSocketSender sender; + + AckFrameHandler(QwpWebSocketSender sender) { + this.sender = sender; + } + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + sender.sawBinaryAck = true; + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + throw new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + } + if (!sender.ackResponse.readFrom(payloadPtr, payloadLen)) { + throw new LineSenderException("Failed to parse ACK response"); + } + } + + @Override + public void onClose(int code, String reason) { + throw new LineSenderException("WebSocket closed while waiting for ACK: " + reason); + } + } } From 5f54bc3588a060bbd2b17e621bda546d045e618e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 12:21:13 +0100 Subject: [PATCH 046/230] Wait for server ACKs on close() in async mode The close() method in async mode was only waiting for pending batches to be written to the wire (via sendQueue.close()), but did not wait for the server to acknowledge receipt. This caused data loss when close() was called without an explicit flush(), since the connection was torn down before the server finished processing. Add sendQueue.flush() and inFlightWindow.awaitEmpty() before sendQueue.close() to match the behavior of flush(). Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index b597d34..13b0206 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -478,6 +478,13 @@ public void close() { if (activeBuffer != null && activeBuffer.hasData()) { sealAndSwapBuffer(); } + // Wait for all batches to be sent and acknowledged before closing + if (sendQueue != null) { + sendQueue.flush(); + } + if (inFlightWindow != null) { + inFlightWindow.awaitEmpty(); + } if (sendQueue != null) { sendQueue.close(); } From b34ba1ecc1d841b1229077f124f3a338b649602b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 12:46:07 +0100 Subject: [PATCH 047/230] Add GEOHASH support to QWP table buffers Add TYPE_GEOHASH to elementSize(), allocateStorage(), and addNull() so geohash values are stored as longs (8 bytes) with -1L as the null sentinel. Also add an explicit case in QwpConstants.getFixedTypeSize() to document that GEOHASH is intentionally variable-width on the wire. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/protocol/QwpConstants.java | 2 ++ .../questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 34f9b2d..54d6fdd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -326,6 +326,8 @@ public static int getFixedTypeSize(byte typeCode) { case TYPE_LONG256: case TYPE_DECIMAL256: return 32; + case TYPE_GEOHASH: + return -1; // Variable width: varint precision + packed values default: return -1; // Variable width } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 4e47c1b..1999fb6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -272,6 +272,7 @@ static int elementSize(byte type) { case TYPE_SYMBOL: case TYPE_FLOAT: return 4; + case TYPE_GEOHASH: case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: @@ -760,6 +761,9 @@ public void addNull() { case TYPE_INT: dataBuffer.putInt(0); break; + case TYPE_GEOHASH: + dataBuffer.putLong(-1L); + break; case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: @@ -1142,6 +1146,7 @@ private void allocateStorage(byte type) { case TYPE_INT: dataBuffer = new OffHeapAppendMemory(64); break; + case TYPE_GEOHASH: case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: From cf64a41c190b40a86a79bbf73f6c595b06d1cc46 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 13:05:13 +0100 Subject: [PATCH 048/230] Use ChaCha20 CSPRNG for WebSocket masking keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the XorShift128 Rnd PRNG with a new ChaCha20-based SecureRnd for WebSocket frame mask key generation. The previous implementation seeded Rnd with System.nanoTime() and System.currentTimeMillis(), which is predictable and does not meet RFC 6455 Section 5.3's requirement for strong entropy. SecureRnd implements ChaCha20 in counter mode (RFC 7539), seeded once from java.security.SecureRandom at construction time. After initialization there are zero heap allocations — all state lives in two pre-allocated int[16] arrays. Each ChaCha20 block yields 16 mask keys, so the amortized cost is minimal. Includes a known-answer test using the RFC 7539 Section 2.3.2 test vector to verify correctness of the ChaCha20 implementation. Co-Authored-By: Claude Opus 4.6 --- .../http/client/WebSocketSendBuffer.java | 6 +- .../cutlass/qwp/client/WebSocketChannel.java | 10 +- .../java/io/questdb/client/std/SecureRnd.java | 185 ++++++++++++++++++ .../client/test/std/SecureRndTest.java | 120 ++++++++++++ 4 files changed, 313 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/std/SecureRnd.java create mode 100644 core/src/test/java/io/questdb/client/test/std/SecureRndTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 66cdf13..5da1c43 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -31,7 +31,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Numbers; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.Rnd; +import io.questdb.client.std.SecureRnd; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; @@ -63,7 +63,7 @@ public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable { private static final int MAX_HEADER_SIZE = 14; private final FrameInfo frameInfo = new FrameInfo(); private final int maxBufferSize; - private final Rnd rnd; + private final SecureRnd rnd; private int bufCapacity; private long bufPtr; private int frameStartOffset; // Where current frame's reserved header starts @@ -99,7 +99,7 @@ public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) { this.writePos = 0; this.frameStartOffset = 0; this.payloadStartOffset = 0; - this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.rnd = new SecureRnd(); } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java index da64ca9..d3965fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java @@ -32,7 +32,7 @@ import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.Rnd; +import io.questdb.client.std.SecureRnd; import io.questdb.client.std.Unsafe; import javax.net.SocketFactory; @@ -76,8 +76,8 @@ public class WebSocketChannel implements QuietCloseable { private final String host; private final String path; private final int port; - // Random for mask key generation - private final Rnd rnd; + // Random for mask key generation (ChaCha20-based CSPRNG, RFC 6455 Section 5.3) + private final SecureRnd rnd; private final boolean tlsEnabled; private final boolean tlsValidationEnabled; private boolean closed; @@ -141,7 +141,7 @@ public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEna this.tlsValidationEnabled = tlsValidationEnabled; this.frameParser = new WebSocketFrameParser(); - this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.rnd = new SecureRnd(); // Allocate native buffers this.sendBufferSize = DEFAULT_BUFFER_SIZE; @@ -380,7 +380,7 @@ private void performHandshake() throws IOException { // Generate random key (16 bytes, base64 encoded = 24 chars) byte[] keyBytes = new byte[16]; for (int i = 0; i < 16; i++) { - keyBytes[i] = (byte) rnd.nextInt(256); + keyBytes[i] = (byte) rnd.nextInt(); } String key = Base64.getEncoder().encodeToString(keyBytes); diff --git a/core/src/main/java/io/questdb/client/std/SecureRnd.java b/core/src/main/java/io/questdb/client/std/SecureRnd.java new file mode 100644 index 0000000..2ceef88 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/SecureRnd.java @@ -0,0 +1,185 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import java.security.SecureRandom; + +/** + * Zero-GC cryptographically secure random number generator based on ChaCha20 + * in counter mode (RFC 7539). Seeded once from {@link SecureRandom} at + * construction time, then produces unpredictable output with no heap + * allocations. + *

    + * Each {@link #nextInt()} call returns one 32-bit word from the ChaCha20 + * keystream. A single block computation yields 16 words, so the amortized + * cost is one ChaCha20 block per 16 calls. + */ +public class SecureRnd { + + // "expand 32-byte k" in little-endian + private static final int CONSTANT_0 = 0x61707865; + private static final int CONSTANT_1 = 0x3320646e; + private static final int CONSTANT_2 = 0x79622d32; + private static final int CONSTANT_3 = 0x6b206574; + + private final int[] output = new int[16]; + private final int[] state = new int[16]; + private int outputPos = 16; // forces block computation on first call + + /** + * Creates a new instance seeded from {@link SecureRandom}. + */ + public SecureRnd() { + SecureRandom seed = new SecureRandom(); + byte[] seedBytes = new byte[44]; // 32 (key) + 12 (nonce) + seed.nextBytes(seedBytes); + init(seedBytes, 0); + } + + /** + * Creates a new instance with an explicit key, nonce, and initial counter. + * Useful for testing with known RFC 7539 test vectors. + * + * @param key 32-byte key + * @param nonce 12-byte nonce + * @param counter initial block counter value + */ + public SecureRnd(byte[] key, byte[] nonce, int counter) { + byte[] seedBytes = new byte[44]; + System.arraycopy(key, 0, seedBytes, 0, 32); + System.arraycopy(nonce, 0, seedBytes, 32, 12); + init(seedBytes, counter); + } + + /** + * Returns the next cryptographically secure random int. + */ + public int nextInt() { + if (outputPos >= 16) { + computeBlock(); + outputPos = 0; + } + return output[outputPos++]; + } + + private void computeBlock() { + int x0 = state[0], x1 = state[1], x2 = state[2], x3 = state[3]; + int x4 = state[4], x5 = state[5], x6 = state[6], x7 = state[7]; + int x8 = state[8], x9 = state[9], x10 = state[10], x11 = state[11]; + int x12 = state[12], x13 = state[13], x14 = state[14], x15 = state[15]; + + for (int i = 0; i < 10; i++) { + // Column rounds + x0 += x4; x12 ^= x0; x12 = Integer.rotateLeft(x12, 16); + x8 += x12; x4 ^= x8; x4 = Integer.rotateLeft(x4, 12); + x0 += x4; x12 ^= x0; x12 = Integer.rotateLeft(x12, 8); + x8 += x12; x4 ^= x8; x4 = Integer.rotateLeft(x4, 7); + + x1 += x5; x13 ^= x1; x13 = Integer.rotateLeft(x13, 16); + x9 += x13; x5 ^= x9; x5 = Integer.rotateLeft(x5, 12); + x1 += x5; x13 ^= x1; x13 = Integer.rotateLeft(x13, 8); + x9 += x13; x5 ^= x9; x5 = Integer.rotateLeft(x5, 7); + + x2 += x6; x14 ^= x2; x14 = Integer.rotateLeft(x14, 16); + x10 += x14; x6 ^= x10; x6 = Integer.rotateLeft(x6, 12); + x2 += x6; x14 ^= x2; x14 = Integer.rotateLeft(x14, 8); + x10 += x14; x6 ^= x10; x6 = Integer.rotateLeft(x6, 7); + + x3 += x7; x15 ^= x3; x15 = Integer.rotateLeft(x15, 16); + x11 += x15; x7 ^= x11; x7 = Integer.rotateLeft(x7, 12); + x3 += x7; x15 ^= x3; x15 = Integer.rotateLeft(x15, 8); + x11 += x15; x7 ^= x11; x7 = Integer.rotateLeft(x7, 7); + + // Diagonal rounds + x0 += x5; x15 ^= x0; x15 = Integer.rotateLeft(x15, 16); + x10 += x15; x5 ^= x10; x5 = Integer.rotateLeft(x5, 12); + x0 += x5; x15 ^= x0; x15 = Integer.rotateLeft(x15, 8); + x10 += x15; x5 ^= x10; x5 = Integer.rotateLeft(x5, 7); + + x1 += x6; x12 ^= x1; x12 = Integer.rotateLeft(x12, 16); + x11 += x12; x6 ^= x11; x6 = Integer.rotateLeft(x6, 12); + x1 += x6; x12 ^= x1; x12 = Integer.rotateLeft(x12, 8); + x11 += x12; x6 ^= x11; x6 = Integer.rotateLeft(x6, 7); + + x2 += x7; x13 ^= x2; x13 = Integer.rotateLeft(x13, 16); + x8 += x13; x7 ^= x8; x7 = Integer.rotateLeft(x7, 12); + x2 += x7; x13 ^= x2; x13 = Integer.rotateLeft(x13, 8); + x8 += x13; x7 ^= x8; x7 = Integer.rotateLeft(x7, 7); + + x3 += x4; x14 ^= x3; x14 = Integer.rotateLeft(x14, 16); + x9 += x14; x4 ^= x9; x4 = Integer.rotateLeft(x4, 12); + x3 += x4; x14 ^= x3; x14 = Integer.rotateLeft(x14, 8); + x9 += x14; x4 ^= x9; x4 = Integer.rotateLeft(x4, 7); + } + + // Feed-forward: add original state + output[0] = x0 + state[0]; + output[1] = x1 + state[1]; + output[2] = x2 + state[2]; + output[3] = x3 + state[3]; + output[4] = x4 + state[4]; + output[5] = x5 + state[5]; + output[6] = x6 + state[6]; + output[7] = x7 + state[7]; + output[8] = x8 + state[8]; + output[9] = x9 + state[9]; + output[10] = x10 + state[10]; + output[11] = x11 + state[11]; + output[12] = x12 + state[12]; + output[13] = x13 + state[13]; + output[14] = x14 + state[14]; + output[15] = x15 + state[15]; + + // Increment block counter + state[12]++; + } + + private void init(byte[] seedBytes, int counter) { + state[0] = CONSTANT_0; + state[1] = CONSTANT_1; + state[2] = CONSTANT_2; + state[3] = CONSTANT_3; + + // Key: 8 little-endian ints from seedBytes[0..31] + for (int i = 0; i < 8; i++) { + int off = i * 4; + state[4 + i] = (seedBytes[off] & 0xFF) + | ((seedBytes[off + 1] & 0xFF) << 8) + | ((seedBytes[off + 2] & 0xFF) << 16) + | ((seedBytes[off + 3] & 0xFF) << 24); + } + + state[12] = counter; + + // Nonce: 3 little-endian ints from seedBytes[32..43] + for (int i = 0; i < 3; i++) { + int off = 32 + i * 4; + state[13 + i] = (seedBytes[off] & 0xFF) + | ((seedBytes[off + 1] & 0xFF) << 8) + | ((seedBytes[off + 2] & 0xFF) << 16) + | ((seedBytes[off + 3] & 0xFF) << 24); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java new file mode 100644 index 0000000..6c04837 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java @@ -0,0 +1,120 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.std; + +import io.questdb.client.std.SecureRnd; +import org.junit.Assert; +import org.junit.Test; + +public class SecureRndTest { + + @Test + public void testConsecutiveCallsProduceDifferentValues() { + SecureRnd rnd = new SecureRnd(); + int prev = rnd.nextInt(); + boolean foundDifferent = false; + for (int i = 0; i < 100; i++) { + int next = rnd.nextInt(); + if (next != prev) { + foundDifferent = true; + break; + } + prev = next; + } + Assert.assertTrue("Expected different values from consecutive calls", foundDifferent); + } + + @Test + public void testDifferentInstancesProduceDifferentSequences() { + SecureRnd rnd1 = new SecureRnd(); + SecureRnd rnd2 = new SecureRnd(); + boolean foundDifferent = false; + for (int i = 0; i < 16; i++) { + if (rnd1.nextInt() != rnd2.nextInt()) { + foundDifferent = true; + break; + } + } + Assert.assertTrue("Two SecureRnd instances should produce different sequences", foundDifferent); + } + + @Test + public void testMultipleBlocksDoNotRepeat() { + SecureRnd rnd = new SecureRnd(); + // Consume more than one block (16 ints) to trigger block counter increment + int[] first16 = new int[16]; + for (int i = 0; i < 16; i++) { + first16[i] = rnd.nextInt(); + } + // Next 16 should be from a different block + boolean foundDifferent = false; + for (int i = 0; i < 16; i++) { + if (rnd.nextInt() != first16[i]) { + foundDifferent = true; + break; + } + } + Assert.assertTrue("Second block should differ from first", foundDifferent); + } + + // RFC 7539 Section 2.3.2 known-answer test + @Test + public void testRfc7539Section232TestVector() { + // Key: 00:01:02:03:...:1f + byte[] key = new byte[32]; + for (int i = 0; i < 32; i++) { + key[i] = (byte) i; + } + + // Nonce: 00:00:00:09:00:00:00:4a:00:00:00:00 + byte[] nonce = { + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x4a, + 0x00, 0x00, 0x00, 0x00 + }; + + // Block counter = 1 + SecureRnd rnd = new SecureRnd(key, nonce, 1); + + // Expected output words (ChaCha state after adding original input) + // from RFC 7539 Section 2.3.2 + int[] expected = { + 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, (int) 0xc47120a3, + (int) 0xc7f4d1c7, 0x0368c033, (int) 0x9aaa2204, 0x4e6cd4c3, + 0x466482d2, 0x09aa9f07, 0x05d7c214, (int) 0xa2028bd9, + (int) 0xd19c12b5, (int) 0xb94e16de, (int) 0xe883d0cb, 0x4e3c50a2, + }; + + for (int i = 0; i < 16; i++) { + int actual = rnd.nextInt(); + Assert.assertEquals( + "Mismatch at word " + i + ": expected 0x" + Integer.toHexString(expected[i]) + + " but got 0x" + Integer.toHexString(actual), + expected[i], + actual + ); + } + } +} From b6609cd95bd39197a325144eb7f2e18b61feaa62 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 13:56:56 +0100 Subject: [PATCH 049/230] Simplify QwpBitReader.alignToByte() Remove the redundant bitsInBuffer % 8 outer guard. Since ensureBits() always loads whole bytes, the invariant (totalBitsRead + bitsInBuffer) % 8 == 0 always holds, making bitsInBuffer % 8 and totalBitsRead % 8 equivalent checks. The simplified version uses only totalBitsRead % 8 which more clearly expresses the intent. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/protocol/QwpBitReader.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index 9241d70..944e4d9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -68,15 +68,9 @@ public QwpBitReader() { * @throws IllegalStateException if alignment fails */ public void alignToByte() { - int bitsToSkip = bitsInBuffer % 8; - if (bitsToSkip != 0) { - // We need to skip the remaining bits in the current partial byte - // But since we read in byte chunks, bitsInBuffer should be a multiple of 8 - // minus what we've consumed. The remainder in the conceptual stream is: - int remainder = (int) (totalBitsRead % 8); - if (remainder != 0) { - skipBits(8 - remainder); - } + int remainder = (int) (totalBitsRead % 8); + if (remainder != 0) { + skipBits(8 - remainder); } } From 8d16e0fef7a6744434a25cc3091bd523dc61cef3 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 14:02:47 +0100 Subject: [PATCH 050/230] Fix putUtf8() lone surrogate encoding mismatch utf8Length() correctly counts lone surrogates as 1 byte ('?' replacement), but putUtf8() let them fall through to the BMP 3-byte encoding path. This 2-byte-per-surrogate discrepancy corrupts varint-prefixed string lengths written by putString(). Add a Character.isSurrogate() check before the 3-byte BMP branch in all three putUtf8() implementations: NativeBufferWriter, WebSocketSendBuffer, and OffHeapAppendMemory. Add tests verifying lone high/low surrogates write 1 byte and that putUtf8() and utf8Length() agree for all surrogate edge cases. Co-Authored-By: Claude Opus 4.6 --- .../http/client/WebSocketSendBuffer.java | 2 ++ .../qwp/client/NativeBufferWriter.java | 2 ++ .../qwp/protocol/OffHeapAppendMemory.java | 2 ++ .../qwp/client/NativeBufferWriterTest.java | 34 +++++++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 5da1c43..6bb280c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -364,6 +364,8 @@ public void putUtf8(String value) { putByte((byte) '?'); i--; } + } else if (Character.isSurrogate(c)) { + putByte((byte) '?'); } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 1e00e12..3230b4c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -255,6 +255,8 @@ public void putUtf8(String value) { putByte((byte) '?'); i--; } + } else if (Character.isSurrogate(c)) { + putByte((byte) '?'); } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index e02398f..7388961 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -162,6 +162,8 @@ public void putUtf8(String value) { Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); i--; } + } else if (Character.isSurrogate(c)) { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); } else { Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12))); Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index a22896d..67a356d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -87,6 +87,40 @@ public void testPutUtf8InvalidSurrogatePair() { } } + @Test + public void testPutUtf8LoneHighSurrogateAtEnd() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uD800"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testPutUtf8LoneLowSurrogate() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uDC00"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testPutUtf8LoneSurrogateMatchesUtf8Length() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // Verify putUtf8 and utf8Length agree for all lone surrogate cases + String[] cases = {"\uD800", "\uDBFF", "\uDC00", "\uDFFF", "\uD800X", "A\uDC00B"}; + for (String s : cases) { + writer.reset(); + writer.putUtf8(s); + assertEquals("length mismatch for: " + s.codePoints() + .mapToObj(cp -> String.format("U+%04X", cp)) + .reduce((a, b) -> a + " " + b).orElse(""), + NativeBufferWriter.utf8Length(s), writer.getPosition()); + } + } + } + @Test public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 From 9254ebf3e409f541cc93c15356b6c5a65b50a34e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 14:48:42 +0100 Subject: [PATCH 051/230] Fix addSymbol() null on non-nullable columns addSymbol() and addSymbolWithGlobalId() incremented size without writing data or incrementing valueCount when value was null and the column was non-nullable. This caused a permanent misalignment between logical row count and physical data. Delegate null values to addNull(), which already handles both nullable (mark in bitmap) and non-nullable (write sentinel) cases correctly. Add a test that verifies size and valueCount stay in sync for non-nullable symbol columns with null values. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 69 +++++++++---------- .../qwp/protocol/QwpTableBufferTest.java | 21 ++++++ 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 1999fb6..b183958 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -836,53 +836,46 @@ public void addString(String value) { public void addSymbol(String value) { if (value == null) { - if (nullable) { - ensureNullCapacity(size + 1); - markNull(size); - } - } else { - ensureNullBitmapForNonNull(); - int idx = symbolDict.get(value); - if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - idx = symbolList.size(); - symbolDict.put(value, idx); - symbolList.add(value); - } - dataBuffer.putInt(idx); - valueCount++; + addNull(); + return; } + ensureNullBitmapForNonNull(); + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + idx = symbolList.size(); + symbolDict.put(value, idx); + symbolList.add(value); + } + dataBuffer.putInt(idx); + valueCount++; size++; } public void addSymbolWithGlobalId(String value, int globalId) { if (value == null) { - if (nullable) { - ensureNullCapacity(size + 1); - markNull(size); - } - size++; - } else { - ensureNullBitmapForNonNull(); - int localIdx = symbolDict.get(value); - if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - localIdx = symbolList.size(); - symbolDict.put(value, localIdx); - symbolList.add(value); - } - dataBuffer.putInt(localIdx); - - if (auxBuffer == null) { - auxBuffer = new OffHeapAppendMemory(64); - } - auxBuffer.putInt(globalId); + addNull(); + return; + } + ensureNullBitmapForNonNull(); + int localIdx = symbolDict.get(value); + if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + localIdx = symbolList.size(); + symbolDict.put(value, localIdx); + symbolList.add(value); + } + dataBuffer.putInt(localIdx); - if (globalId > maxGlobalSymbolId) { - maxGlobalSymbolId = globalId; - } + if (auxBuffer == null) { + auxBuffer = new OffHeapAppendMemory(64); + } + auxBuffer.putInt(globalId); - valueCount++; - size++; + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; } + + valueCount++; + size++; } public void addUuid(long high, long low) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index e4ac0d7..7e74ff7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -34,6 +34,27 @@ public class QwpTableBufferTest { + @Test + public void testAddSymbolNullOnNonNullableColumn() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, false); + col.addSymbol("server1"); + table.nextRow(); + + // Null on a non-nullable column must write a sentinel value, + // keeping size and valueCount in sync + col.addSymbol(null); + table.nextRow(); + + col.addSymbol("server2"); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + // For non-nullable columns, every row must have a physical value + assertEquals(col.getSize(), col.getValueCount()); + } + } + @Test public void testCancelRowRewindsDoubleArrayOffsets() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { From 391fce46fdf13961a42dfe518d5a57523bd30f9f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 14:58:30 +0100 Subject: [PATCH 052/230] Fix addNull() for array types on non-nullable columns QwpTableBuffer.addNull() had no handling for TYPE_DOUBLE_ARRAY and TYPE_LONG_ARRAY in the non-nullable branch. This caused valueCount and size to advance without writing array metadata (dims, shapes), corrupting array index tracking for subsequent rows. The fix writes an empty 1D array (dims=1, shape=0, no data elements) as the sentinel value, keeping dims/shapes/data offsets consistent. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 6 ++ .../qwp/protocol/QwpTableBufferTest.java | 72 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index b183958..119bbbc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -806,6 +806,12 @@ public void addNull() { dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + ensureArrayCapacity(1, 0); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = 0; + break; } valueCount++; size++; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 7e74ff7..1c294d0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -34,6 +34,78 @@ public class QwpTableBufferTest { + @Test + public void testAddDoubleArrayNullOnNonNullableColumn() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + // Row 0: real array + col.addDoubleArray(new double[]{1.0, 2.0}); + table.nextRow(); + + // Row 1: null on non-nullable — must write empty array metadata + col.addDoubleArray((double[]) null); + table.nextRow(); + + // Row 2: real array + col.addDoubleArray(new double[]{3.0, 4.0}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, encoded, 0.0); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + } + + @Test + public void testAddLongArrayNullOnNonNullableColumn() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + // Row 0: real array + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + // Row 1: null on non-nullable — must write empty array metadata + col.addLongArray((long[]) null); + table.nextRow(); + + // Row 2: real array + col.addLongArray(new long[]{30, 40}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 30, 40}, encoded); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + } + @Test public void testAddSymbolNullOnNonNullableColumn() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { From a92e7b42ef54f7548c5a0ed0ddb1d2c571315ccb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:09:51 +0100 Subject: [PATCH 053/230] Fix sendCloseFrame/sendPing clobbering sendBuffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sendCloseFrame() and sendPing() used the main sendBuffer to build and send control frames. If the caller had an in-progress data frame in sendBuffer (obtained via getSendBuffer()), these methods would destroy it by calling sendBuffer.reset(). Switch both methods to use controlFrameBuffer, which already exists for exactly this purpose — sendCloseFrameEcho() and sendPongFrame() were already using it correctly. Add unit tests that verify the sendBuffer is preserved across sendCloseFrame() and sendPing() calls. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 16 +- .../http/client/WebSocketClientTest.java | 137 ++++++++++++++++++ 2 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 32d79dd..9c6ca38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -312,12 +312,12 @@ public void sendBinary(long dataPtr, int length) { * Sends a close frame. */ public void sendCloseFrame(int code, String reason, int timeout) { - sendBuffer.reset(); - WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason); + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, reason); try { - doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout); } finally { - sendBuffer.reset(); + controlFrameBuffer.reset(); } } @@ -344,10 +344,10 @@ public void sendFrame(WebSocketSendBuffer.FrameInfo frame) { */ public void sendPing(int timeout) { checkConnected(); - sendBuffer.reset(); - WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePingFrame(); - doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); - sendBuffer.reset(); + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePingFrame(); + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + controlFrameBuffer.reset(); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java new file mode 100644 index 0000000..34dd0bd --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -0,0 +1,137 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; + +public class WebSocketClientTest { + + @Test + public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (StubWebSocketClient client = new StubWebSocketClient()) { + WebSocketSendBuffer sendBuffer = client.getSendBuffer(); + + // User starts building a data frame + sendBuffer.beginFrame(); + sendBuffer.putLong(0xDEADBEEFL); + int posBeforeClose = sendBuffer.getWritePos(); + Assert.assertTrue("sendBuffer should have data", posBeforeClose > 0); + + // sendCloseFrame() should use controlFrameBuffer, not sendBuffer + try { + client.sendCloseFrame(1000, null, 1000); + } catch (HttpClientException ignored) { + // Expected: doSend() fails because there's no real socket + } + + // Verify sendBuffer was NOT clobbered + Assert.assertEquals( + "sendCloseFrame() must not reset the main sendBuffer", + posBeforeClose, + sendBuffer.getWritePos() + ); + } + }); + } + + @Test + public void testSendPingDoesNotClobberSendBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (StubWebSocketClient client = new StubWebSocketClient()) { + // Set upgraded=true so checkConnected() passes + setField(client, "upgraded", true); + + WebSocketSendBuffer sendBuffer = client.getSendBuffer(); + + // User starts building a data frame + sendBuffer.beginFrame(); + sendBuffer.putLong(0xCAFEBABEL); + int posBeforePing = sendBuffer.getWritePos(); + Assert.assertTrue("sendBuffer should have data", posBeforePing > 0); + + // sendPing() should use controlFrameBuffer, not sendBuffer + try { + client.sendPing(1000); + } catch (HttpClientException ignored) { + // Expected: doSend() fails because there's no real socket + } + + // Verify sendBuffer was NOT clobbered + Assert.assertEquals( + "sendPing() must not reset the main sendBuffer", + posBeforePing, + sendBuffer.getWritePos() + ); + } + }); + } + + private static void setField(Object obj, String fieldName, Object value) throws Exception { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(obj, value); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } + + /** + * Minimal concrete WebSocketClient that throws on any I/O, + * allowing us to test buffer management without a real socket. + */ + private static class StubWebSocketClient extends WebSocketClient { + + StubWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new HttpClientException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} From c6278238a8e5f6358eb6796389458af6e6073c65 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:22:06 +0100 Subject: [PATCH 054/230] Fix WebSocketSendBuffer.grow() integer overflow Numbers.ceilPow2(int) returns 2^30 for inputs between 2^30+1 and Integer.MAX_VALUE due to internal overflow handling. This caused grow() to allocate a buffer smaller than the required capacity. Fix by taking the max of ceilPow2's result and the raw required capacity before clamping to maxBufferSize. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/cutlass/http/client/WebSocketSendBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 6bb280c..e4d7ba5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -527,7 +527,7 @@ private void grow(long requiredCapacity) { .put(']'); } int newCapacity = Math.min( - Numbers.ceilPow2((int) requiredCapacity), + Math.max(Numbers.ceilPow2((int) requiredCapacity), (int) requiredCapacity), maxBufferSize ); bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT); From 4f4f28b616d78ba6157f2ae036833a5814acb916 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:27:48 +0100 Subject: [PATCH 055/230] Fix readFrom() leaking stale error messages When readFrom() parsed an error response where status != OK and the buffer was large enough to enter the outer branch (length > offset + 2), but msgLen was 0, the inner condition (msgLen > 0) failed without clearing errorMessage. On reused WebSocketResponse objects this left the error message from a previous parse visible to callers. Add an else branch to the inner if that sets errorMessage = null when msgLen is 0 or the message bytes are truncated. Add a regression test that parses an error-with-message followed by an error-with-empty-message on the same object and asserts errorMessage is null. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/WebSocketResponse.java | 2 + .../qwp/client/WebSocketChannelTest.java | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index e1c1e6b..169f419 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -211,6 +211,8 @@ public boolean readFrom(long ptr, int length) { } errorMessage = new String(msgBytes, StandardCharsets.UTF_8); offset += msgLen; + } else { + errorMessage = null; } } else { errorMessage = null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java index 47fe527..4616a7a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.WebSocketChannel; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.std.MemoryTag; @@ -135,6 +136,44 @@ public void testBinaryRoundTripSmallPayload() throws Exception { TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); } + @Test + public void testResponseReadFromEmptyErrorClearsStaleMessage() { + // First, parse an error response WITH an error message + WebSocketResponse response = new WebSocketResponse(); + WebSocketResponse errorWithMsg = WebSocketResponse.error(42, WebSocketResponse.STATUS_PARSE_ERROR, "bad input"); + int size1 = errorWithMsg.serializedSize(); + long ptr = Unsafe.malloc(size1, MemoryTag.NATIVE_DEFAULT); + try { + errorWithMsg.writeTo(ptr); + Assert.assertTrue(response.readFrom(ptr, size1)); + Assert.assertEquals("bad input", response.getErrorMessage()); + } finally { + Unsafe.free(ptr, size1, MemoryTag.NATIVE_DEFAULT); + } + + // Now, parse an error response with an EMPTY error message (msgLen=0) + // but with a buffer larger than MIN_ERROR_RESPONSE_SIZE. This triggers + // the path where the outer if (length > offset + 2) is true, but the + // inner if (msgLen > 0) is false, leaving errorMessage stale. + int size2 = WebSocketResponse.MIN_ERROR_RESPONSE_SIZE + 1; + ptr = Unsafe.malloc(size2, MemoryTag.NATIVE_DEFAULT); + try { + int offset = 0; + Unsafe.getUnsafe().putByte(ptr + offset, WebSocketResponse.STATUS_WRITE_ERROR); + offset += 1; + Unsafe.getUnsafe().putLong(ptr + offset, 99L); + offset += 8; + Unsafe.getUnsafe().putShort(ptr + offset, (short) 0); // msgLen = 0 + + Assert.assertTrue(response.readFrom(ptr, size2)); + Assert.assertEquals(WebSocketResponse.STATUS_WRITE_ERROR, response.getStatus()); + Assert.assertEquals(99L, response.getSequence()); + Assert.assertNull("errorMessage should be null for empty error message", response.getErrorMessage()); + } finally { + Unsafe.free(ptr, size2, MemoryTag.NATIVE_DEFAULT); + } + } + /** * Calls receiveFrame in a loop to handle the case where doReceiveFrame * needs multiple reads to assemble a complete frame (e.g. header and From c9849192ef80ffed49d28a863e053039bfa88aed Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:44:12 +0100 Subject: [PATCH 056/230] Fix reset() to clear all table buffers QwpWebSocketSender.reset() only reset the current table buffer, leaving other tables' data intact and pendingRowCount nonzero. The Sender.reset() contract requires all pending state to be discarded. Now iterate every table buffer in the map, zero pendingRowCount and firstPendingRowTimeNanos, and clear the current-table and cached-column references. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 23 ++++- .../client/QwpWebSocketSenderResetTest.java | 96 +++++++++++++++++++ 2 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 13b0206..382107a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -710,6 +710,14 @@ public int getMaxSentSymbolId() { return maxSentSymbolId; } + /** + * Returns the number of pending rows not yet flushed. + * For testing. + */ + public int getPendingRowCount() { + return pendingRowCount; + } + /** * Registers a symbol in the global dictionary and returns its ID. * For use with fast-path column buffer access. @@ -866,9 +874,20 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { @Override public void reset() { checkNotClosed(); - if (currentTableBuffer != null) { - currentTableBuffer.reset(); + // Reset ALL table buffers, not just the current one + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.reset(); + } } + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + currentTableBuffer = null; + currentTableName = null; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java new file mode 100644 index 0000000..ea1cd6c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java @@ -0,0 +1,96 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.temporal.ChronoUnit; + +/** + * Verifies that {@link QwpWebSocketSender#reset()} discards all pending state, + * not just the current table buffer. + */ +public class QwpWebSocketSenderResetTest extends AbstractTest { + + @Test + public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Use high autoFlushRows to prevent auto-flush during the test + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 10_000, 10_000_000, 0, 1, 16 + ); + try { + // Bypass ensureConnected() — mark as connected, leave client null + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Buffer rows into two different tables via the fluent API + sender.table("t1") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + sender.table("t2") + .longColumn("y", 2) + .at(2, ChronoUnit.MICROS); + + // Verify data is buffered + QwpTableBuffer t1 = sender.getTableBuffer("t1"); + QwpTableBuffer t2 = sender.getTableBuffer("t2"); + Assert.assertEquals("t1 should have 1 row before reset", 1, t1.getRowCount()); + Assert.assertEquals("t2 should have 1 row before reset", 1, t2.getRowCount()); + Assert.assertEquals("pendingRowCount should be 2 before reset", 2, sender.getPendingRowCount()); + + // Select t1 as the current table + sender.table("t1"); + + // Call reset — per the Sender contract this should discard + // ALL pending state, not just the current table + sender.reset(); + + // Both table buffers should be cleared + Assert.assertEquals("t1 row count should be 0 after reset", 0, t1.getRowCount()); + Assert.assertEquals("t2 row count should be 0 after reset", 0, t2.getRowCount()); + + // Pending row count should be zeroed + Assert.assertEquals("pendingRowCount should be 0 after reset", 0, sender.getPendingRowCount()); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field f = target.getClass().getDeclaredField(fieldName); + f.setAccessible(true); + f.set(target, value); + } +} From 93dce206cfcc1697bf3d4eb3dc10c784380ad2d8 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:56:29 +0100 Subject: [PATCH 057/230] Fix ensureBits() 64-bit buffer overflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureBits() loaded bytes into the 64-bit bitBuffer without checking whether all 8 bits of the incoming byte would actually fit. When bitsInBuffer exceeded 56, the left-shift lost high bits that overflowed position 63, and bitsInBuffer could grow past 64 — silently corrupting the buffer. Add a bitsInBuffer <= 56 guard to the while loop so we never attempt to load a byte when fewer than 8 free bit positions remain. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index 944e4d9..c1e2ec7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -316,7 +316,7 @@ public void skipBits(int numBits) { * @return true if sufficient bits available, false otherwise */ private boolean ensureBits(int bitsNeeded) { - while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) { + while (bitsInBuffer < bitsNeeded && bitsInBuffer <= 56 && currentAddress < endAddress) { byte b = Unsafe.getUnsafe().getByte(currentAddress++); bitBuffer |= (long) (b & 0xFF) << bitsInBuffer; bitsInBuffer += 8; From 1180216dd29c1869719cd13701653a3c2f97542a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 16:30:15 +0100 Subject: [PATCH 058/230] Add bounds check to NativeBufferWriter.skip() skip() advanced the position without calling ensureCapacity(), so a skip that exceeded the current buffer capacity would let subsequent writes corrupt native memory past the allocation. Add the missing ensureCapacity(bytes) call, matching the existing pattern in WebSocketSendBuffer.skip() and OffHeapAppendMemory.skip(). Add a regression test that skips past the initial capacity and verifies the buffer grows. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/NativeBufferWriter.java | 1 + .../cutlass/qwp/client/NativeBufferWriterTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 3230b4c..19f0ae3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -293,6 +293,7 @@ public void reset() { */ @Override public void skip(int bytes) { + ensureCapacity(bytes); position += bytes; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 67a356d..c3b8991 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -75,6 +75,20 @@ public void testSkipAdvancesPosition() { } } + @Test + public void testSkipBeyondCapacityGrowsBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // skip past the 16-byte buffer — must grow, not corrupt memory + writer.skip(32); + assertEquals(32, writer.getPosition()); + assertTrue(writer.getCapacity() >= 32); + // writing after the skip must also succeed + writer.putInt(0xCAFE); + assertEquals(36, writer.getPosition()); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + } + } + @Test public void testPutUtf8InvalidSurrogatePair() { try (NativeBufferWriter writer = new NativeBufferWriter(64)) { From c55d9820c9df69d00edf02d9436c31f7633eca2f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 17:04:32 +0100 Subject: [PATCH 059/230] Invalidate cached timestamp columns on flush flushPendingRows() and flushSync() reset table buffers but did not clear cachedTimestampColumn and cachedTimestampNanosColumn. If the table buffer's columns were ever recreated rather than just data- reset, the stale references would become dangling. Null both fields at the start of each flush method, consistent with what reset() and table() already do. Add QwpWebSocketSenderFlushCacheTest to verify the invariant for both micros and nanos timestamp paths. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 8 + .../QwpWebSocketSenderFlushCacheTest.java | 140 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 382107a..50167f6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1148,6 +1148,10 @@ private void flushPendingRows() { return; } + // Invalidate cached column references — table buffers will be reset below + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + LOG.debug("Flushing pending rows [count={}, tables={}]", pendingRowCount, tableBuffers.size()); // Ensure activeBuffer is ready for writing @@ -1226,6 +1230,10 @@ private void flushSync() { return; } + // Invalidate cached column references — table buffers will be reset below + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); // Encode all table buffers that have data into a single message diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java new file mode 100644 index 0000000..b8ef386 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java @@ -0,0 +1,140 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.temporal.ChronoUnit; + +/** + * Verifies that {@link QwpWebSocketSender} invalidates its cached timestamp + * column references ({@code cachedTimestampColumn} and + * {@code cachedTimestampNanosColumn}) during flush operations. + *

    + * These cached references point into a {@code QwpTableBuffer} whose columns + * are reset by {@code flushSync()} / {@code flushPendingRows()}. If the cache + * is not cleared, subsequent rows may write through a stale reference. + *

    + * The test uses {@code autoFlushRows=1} so that every row triggers a flush + * inside {@code sendRow()}. The flush itself fails (no real connection), but + * the cache must be invalidated before the send is attempted. + * After the failed flush the test clears the table buffer, making any + * surviving stale reference point to a freed {@code ColumnBuffer}. A second + * row is then sent: if the cache was properly invalidated, a fresh column is + * created and the row is buffered normally; if stale, {@code addLong()} hits + * an NPE before {@code sendRow()} / {@code nextRow()}, so the row is never + * counted. + */ +public class QwpWebSocketSenderFlushCacheTest extends AbstractTest { + + @Test + public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 1, 10_000_000, 0, 1, 16 + ); + try { + setConnected(sender, true); + + // Row 1: caches cachedTimestampColumn, then auto-flush + // triggers and fails (no real connection). + try { + sender.table("t") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + } catch (Exception ignored) { + } + + // Clear the table buffer so a stale cached reference now + // points to a freed ColumnBuffer. + QwpTableBuffer tb = sender.getTableBuffer("t"); + tb.clear(); + + // Row 2: with the fix, atMicros() creates a fresh column + // and the row is buffered. Without, addLong() NPEs before + // sendRow()/nextRow() and the row is never counted. + try { + sender.table("t") + .longColumn("x", 2) + .at(2, ChronoUnit.MICROS); + } catch (Exception ignored) { + } + + Assert.assertEquals("row must be buffered when cache is properly invalidated", + 1, tb.getRowCount()); + } finally { + setConnected(sender, false); + sender.close(); + } + }); + } + + @Test + public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 1, 10_000_000, 0, 1, 16 + ); + try { + setConnected(sender, true); + + try { + sender.table("t") + .longColumn("x", 1) + .at(1, ChronoUnit.NANOS); + } catch (Exception ignored) { + } + + QwpTableBuffer tb = sender.getTableBuffer("t"); + tb.clear(); + + try { + sender.table("t") + .longColumn("x", 2) + .at(2, ChronoUnit.NANOS); + } catch (Exception ignored) { + } + + Assert.assertEquals("row must be buffered when cache is properly invalidated", + 1, tb.getRowCount()); + } finally { + setConnected(sender, false); + sender.close(); + } + }); + } + + private static void setConnected(QwpWebSocketSender sender, boolean value) throws Exception { + Field f = QwpWebSocketSender.class.getDeclaredField("connected"); + f.setAccessible(true); + f.set(sender, value); + } +} From ecb595f39aa87d4af9ca01cd567ecb9e337ea3e4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 17:09:12 +0100 Subject: [PATCH 060/230] Fix putBlockOfBytes() long-to-int overflow putBlockOfBytes() accepted a long len parameter but cast it to int before calling ensureCapacity(). When len > Integer.MAX_VALUE, the cast wraps to a negative number, so ensureCapacity() skips the buffer grow, but copyMemory() still uses the original long len, causing a buffer overflow. Validate len fits in int range before casting in both NativeBufferWriter and WebSocketSendBuffer. Use the narrowed int value consistently for ensureCapacity(), copyMemory(), and position update. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketSendBuffer.java | 10 +++++++--- .../cutlass/qwp/client/NativeBufferWriter.java | 10 +++++++--- .../cutlass/qwp/client/NativeBufferWriterTest.java | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index e4d7ba5..afb0d05 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -249,9 +249,13 @@ public void putBlockOfBytes(long from, long len) { if (len <= 0) { return; } - ensureCapacity((int) len); - Vect.memcpy(bufPtr + writePos, from, len); - writePos += (int) len; + if (len > Integer.MAX_VALUE) { + throw new IllegalArgumentException("len exceeds int range: " + len); + } + int intLen = (int) len; + ensureCapacity(intLen); + Vect.memcpy(bufPtr + writePos, from, intLen); + writePos += intLen; } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 19f0ae3..5d8eb9e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -138,9 +138,13 @@ public void patchInt(int offset, int value) { */ @Override public void putBlockOfBytes(long from, long len) { - ensureCapacity((int) len); - Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len); - position += (int) len; + if (len < 0 || len > Integer.MAX_VALUE) { + throw new IllegalArgumentException("len exceeds int range: " + len); + } + int intLen = (int) len; + ensureCapacity(intLen); + Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, intLen); + position += intLen; } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index c3b8991..bfa2bc7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -27,10 +27,12 @@ import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.std.Unsafe; +import org.junit.Assert; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class NativeBufferWriterTest { @@ -89,6 +91,18 @@ public void testSkipBeyondCapacityGrowsBuffer() { } } + @Test + public void testPutBlockOfBytesRejectsLenExceedingIntMax() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + try { + writer.putBlockOfBytes(0, (long) Integer.MAX_VALUE + 1); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("len")); + } + } + } + @Test public void testPutUtf8InvalidSurrogatePair() { try (NativeBufferWriter writer = new NativeBufferWriter(64)) { From 542a667cd0ba199ca5cfc7da1aff6bdfc1087373 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 17:14:37 +0100 Subject: [PATCH 061/230] Merge sender state tests into one class Merge QwpWebSocketSenderResetTest and QwpWebSocketSenderFlushCacheTest into a single QwpWebSocketSenderStateTest class. Both tested QwpWebSocketSender internal state management with the same reflection pattern and superclass. The merged class unifies the setField/ setConnected helpers into one setField method and keeps all three test methods in alphabetical order. Co-Authored-By: Claude Opus 4.6 --- .../client/QwpWebSocketSenderResetTest.java | 96 ------------------- ....java => QwpWebSocketSenderStateTest.java} | 87 ++++++++++++----- 2 files changed, 62 insertions(+), 121 deletions(-) delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java rename core/src/test/java/io/questdb/client/test/cutlass/qwp/client/{QwpWebSocketSenderFlushCacheTest.java => QwpWebSocketSenderStateTest.java} (56%) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java deleted file mode 100644 index ea1cd6c..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client; - -import io.questdb.client.cutlass.qwp.client.InFlightWindow; -import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; -import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.test.AbstractTest; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.lang.reflect.Field; -import java.time.temporal.ChronoUnit; - -/** - * Verifies that {@link QwpWebSocketSender#reset()} discards all pending state, - * not just the current table buffer. - */ -public class QwpWebSocketSenderResetTest extends AbstractTest { - - @Test - public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { - TestUtils.assertMemoryLeak(() -> { - // Use high autoFlushRows to prevent auto-flush during the test - QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 10_000, 10_000_000, 0, 1, 16 - ); - try { - // Bypass ensureConnected() — mark as connected, leave client null - setField(sender, "connected", true); - setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); - - // Buffer rows into two different tables via the fluent API - sender.table("t1") - .longColumn("x", 1) - .at(1, ChronoUnit.MICROS); - sender.table("t2") - .longColumn("y", 2) - .at(2, ChronoUnit.MICROS); - - // Verify data is buffered - QwpTableBuffer t1 = sender.getTableBuffer("t1"); - QwpTableBuffer t2 = sender.getTableBuffer("t2"); - Assert.assertEquals("t1 should have 1 row before reset", 1, t1.getRowCount()); - Assert.assertEquals("t2 should have 1 row before reset", 1, t2.getRowCount()); - Assert.assertEquals("pendingRowCount should be 2 before reset", 2, sender.getPendingRowCount()); - - // Select t1 as the current table - sender.table("t1"); - - // Call reset — per the Sender contract this should discard - // ALL pending state, not just the current table - sender.reset(); - - // Both table buffers should be cleared - Assert.assertEquals("t1 row count should be 0 after reset", 0, t1.getRowCount()); - Assert.assertEquals("t2 row count should be 0 after reset", 0, t2.getRowCount()); - - // Pending row count should be zeroed - Assert.assertEquals("pendingRowCount should be 0 after reset", 0, sender.getPendingRowCount()); - } finally { - setField(sender, "connected", false); - sender.close(); - } - }); - } - - private static void setField(Object target, String fieldName, Object value) throws Exception { - Field f = target.getClass().getDeclaredField(fieldName); - f.setAccessible(true); - f.set(target, value); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java similarity index 56% rename from core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java rename to core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index b8ef386..f67ef61 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.test.AbstractTest; @@ -35,25 +36,14 @@ import java.time.temporal.ChronoUnit; /** - * Verifies that {@link QwpWebSocketSender} invalidates its cached timestamp - * column references ({@code cachedTimestampColumn} and - * {@code cachedTimestampNanosColumn}) during flush operations. - *

    - * These cached references point into a {@code QwpTableBuffer} whose columns - * are reset by {@code flushSync()} / {@code flushPendingRows()}. If the cache - * is not cleared, subsequent rows may write through a stale reference. - *

    - * The test uses {@code autoFlushRows=1} so that every row triggers a flush - * inside {@code sendRow()}. The flush itself fails (no real connection), but - * the cache must be invalidated before the send is attempted. - * After the failed flush the test clears the table buffer, making any - * surviving stale reference point to a freed {@code ColumnBuffer}. A second - * row is then sent: if the cache was properly invalidated, a fresh column is - * created and the row is buffered normally; if stale, {@code addLong()} hits - * an NPE before {@code sendRow()} / {@code nextRow()}, so the row is never - * counted. + * Verifies {@link QwpWebSocketSender} internal state management: + *

      + *
    • {@code reset()} discards all pending state, not just the current table buffer.
      • + *
    • Cached timestamp column references are invalidated during flush operations, + * preventing stale writes through freed {@code ColumnBuffer} instances.
      • + *
    */ -public class QwpWebSocketSenderFlushCacheTest extends AbstractTest { +public class QwpWebSocketSenderStateTest extends AbstractTest { @Test public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { @@ -62,7 +52,7 @@ public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { "localhost", 0, 1, 10_000_000, 0, 1, 16 ); try { - setConnected(sender, true); + setField(sender, "connected", true); // Row 1: caches cachedTimestampColumn, then auto-flush // triggers and fails (no real connection). @@ -91,7 +81,7 @@ public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { Assert.assertEquals("row must be buffered when cache is properly invalidated", 1, tb.getRowCount()); } finally { - setConnected(sender, false); + setField(sender, "connected", false); sender.close(); } }); @@ -104,7 +94,7 @@ public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Except "localhost", 0, 1, 10_000_000, 0, 1, 16 ); try { - setConnected(sender, true); + setField(sender, "connected", true); try { sender.table("t") @@ -126,15 +116,62 @@ public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Except Assert.assertEquals("row must be buffered when cache is properly invalidated", 1, tb.getRowCount()); } finally { - setConnected(sender, false); + setField(sender, "connected", false); sender.close(); } }); } - private static void setConnected(QwpWebSocketSender sender, boolean value) throws Exception { - Field f = QwpWebSocketSender.class.getDeclaredField("connected"); + @Test + public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Use high autoFlushRows to prevent auto-flush during the test + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 10_000, 10_000_000, 0, 1, 16 + ); + try { + // Bypass ensureConnected() — mark as connected, leave client null + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Buffer rows into two different tables via the fluent API + sender.table("t1") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + sender.table("t2") + .longColumn("y", 2) + .at(2, ChronoUnit.MICROS); + + // Verify data is buffered + QwpTableBuffer t1 = sender.getTableBuffer("t1"); + QwpTableBuffer t2 = sender.getTableBuffer("t2"); + Assert.assertEquals("t1 should have 1 row before reset", 1, t1.getRowCount()); + Assert.assertEquals("t2 should have 1 row before reset", 1, t2.getRowCount()); + Assert.assertEquals("pendingRowCount should be 2 before reset", 2, sender.getPendingRowCount()); + + // Select t1 as the current table + sender.table("t1"); + + // Call reset — per the Sender contract this should discard + // ALL pending state, not just the current table + sender.reset(); + + // Both table buffers should be cleared + Assert.assertEquals("t1 row count should be 0 after reset", 0, t1.getRowCount()); + Assert.assertEquals("t2 row count should be 0 after reset", 0, t2.getRowCount()); + + // Pending row count should be zeroed + Assert.assertEquals("pendingRowCount should be 0 after reset", 0, sender.getPendingRowCount()); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field f = target.getClass().getDeclaredField(fieldName); f.setAccessible(true); - f.set(sender, value); + f.set(target, value); } } From 243c03e460e2e562fdfb27be064b6cfb03e53f09 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 09:53:53 +0100 Subject: [PATCH 062/230] Remove unused sendQueueCapacity parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WebSocketSendQueue accepted a queueCapacity parameter in its constructor, validated it, and logged it, but never used it. The actual queue is a single volatile slot (pendingBuffer) by design — matching the double-buffering scheme where at most one sealed buffer is pending while the other is being filled. Remove the parameter from the entire chain: WebSocketSendQueue constructor, QwpWebSocketSender field and factory methods, Sender.LineSenderBuilder API, and all tests and benchmark clients that referenced it. Co-Authored-By: Claude Opus 4.6 --- .../main/java/io/questdb/client/Sender.java | 32 +-------- .../qwp/client/QwpWebSocketSender.java | 33 +++------- .../qwp/client/WebSocketSendQueue.java | 22 +++---- .../line/tcp/v4/QwpAllocationTestClient.java | 14 ++-- .../line/tcp/v4/StacBenchmarkClient.java | 14 ++-- .../LineSenderBuilderWebSocketTest.java | 65 +------------------ .../client/QwpWebSocketSenderStateTest.java | 6 +- 7 files changed, 33 insertions(+), 153 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 755bd78..705f3fe 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -537,7 +537,6 @@ final class LineSenderBuilder { private static final int DEFAULT_MAX_NAME_LEN = 127; private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method - private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16; private static final int DEFAULT_TCP_PORT = 9009; private static final int DEFAULT_WEBSOCKET_PORT = 9000; private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB @@ -596,7 +595,6 @@ public int getTimeout() { private int protocol = PARAMETER_NOT_SET_EXPLICITLY; private int protocolVersion = PARAMETER_NOT_SET_EXPLICITLY; private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY; - private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY; private boolean shouldDestroyPrivKey; private boolean tlsEnabled; private TlsValidationMode tlsValidationMode; @@ -878,7 +876,6 @@ public Sender build() { ? DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis); int actualInFlightWindowSize = inFlightWindowSize == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_IN_FLIGHT_WINDOW_SIZE : inFlightWindowSize; - int actualSendQueueCapacity = sendQueueCapacity == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEND_QUEUE_CAPACITY : sendQueueCapacity; if (asyncMode) { return QwpWebSocketSender.connectAsync( @@ -888,8 +885,7 @@ public Sender build() { actualAutoFlushRows, actualAutoFlushBytes, actualAutoFlushIntervalNanos, - actualInFlightWindowSize, - actualSendQueueCapacity + actualInFlightWindowSize ); } else { return QwpWebSocketSender.connect( @@ -1374,29 +1370,6 @@ public LineSenderBuilder retryTimeoutMillis(int retryTimeoutMillis) { return this; } - /** - * Set the capacity of the send queue for batches waiting to be sent. - *
    - * This is only used when communicating over WebSocket transport with async mode enabled. - *
    - * Default value is 16. - * - * @param capacity send queue capacity - * @return this instance for method chaining - */ - public LineSenderBuilder sendQueueCapacity(int capacity) { - if (this.sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY) { - throw new LineSenderException("send queue capacity was already configured") - .put("[capacity=").put(this.sendQueueCapacity).put("]"); - } - if (capacity < 1) { - throw new LineSenderException("send queue capacity must be positive") - .put("[capacity=").put(capacity).put("]"); - } - this.sendQueueCapacity = capacity; - return this; - } - private static int getValue(CharSequence configurationString, int pos, StringSink sink, String name) { if ((pos = ConfStringParser.value(configurationString, pos, sink)) < 0) { throw new LineSenderException("invalid ").put(name).put(" [error=").put(sink).put("]"); @@ -1802,9 +1775,6 @@ private void validateParameters() { if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { throw new LineSenderException("in-flight window size requires async mode"); } - if (sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { - throw new LineSenderException("send queue capacity requires async mode"); - } } else { throw new LineSenderException("unsupported protocol ") .put("[protocol=").put(protocol).put("]"); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 50167f6..59e890a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -109,7 +109,6 @@ public class QwpWebSocketSender implements Sender { public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 - public static final int DEFAULT_SEND_QUEUE_CAPACITY = WebSocketSendQueue.DEFAULT_QUEUE_CAPACITY; // 16 private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); @@ -126,7 +125,6 @@ public class QwpWebSocketSender implements Sender { // Flow control configuration private final int inFlightWindowSize; private final int port; - private final int sendQueueCapacity; // Track schema hashes that have been sent to the server (for schema reference mode) // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. @@ -173,8 +171,7 @@ private QwpWebSocketSender( int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, - int sendQueueCapacity + int inFlightWindowSize ) { this.host = host; this.port = port; @@ -189,7 +186,6 @@ private QwpWebSocketSender( this.autoFlushBytes = autoFlushBytes; this.autoFlushIntervalNanos = autoFlushIntervalNanos; this.inFlightWindowSize = inFlightWindowSize; - this.sendQueueCapacity = sendQueueCapacity; // Initialize global symbol dictionary for delta encoding this.globalSymbolDictionary = new GlobalSymbolDictionary(); @@ -247,7 +243,7 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - 1, 1 // window=1 for sync behavior, queue=1 (not used) + 1 // window=1 for sync behavior ); sender.ensureConnected(); return sender; @@ -268,7 +264,7 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos) { return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - DEFAULT_IN_FLIGHT_WINDOW_SIZE, DEFAULT_SEND_QUEUE_CAPACITY); + DEFAULT_IN_FLIGHT_WINDOW_SIZE); } /** @@ -281,7 +277,6 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls * @param autoFlushBytes bytes per batch (0 = no limit) * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @param inFlightWindowSize max batches awaiting server ACK (default: 8) - * @param sendQueueCapacity max batches waiting to send (default: 16) * @return connected sender */ public static QwpWebSocketSender connectAsync( @@ -291,13 +286,12 @@ public static QwpWebSocketSender connectAsync( int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, - int sendQueueCapacity + int inFlightWindowSize ) { QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - inFlightWindowSize, sendQueueCapacity + inFlightWindowSize ); sender.ensureConnected(); return sender; @@ -331,7 +325,7 @@ public static QwpWebSocketSender create( QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, bufferSize, 0, 0, 0, - 1, 1 // window=1 for sync behavior + 1 // window=1 for sync behavior ); // TODO: Store auth credentials for connection sender.ensureConnected(); @@ -353,7 +347,7 @@ public static QwpWebSocketSender createForTesting(String host, int port, int inF return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, - inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY + inFlightWindowSize ); // Note: does NOT call ensureConnected() } @@ -367,17 +361,16 @@ public static QwpWebSocketSender createForTesting(String host, int port, int inF * @param autoFlushBytes bytes per batch (0 = no limit) * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async - * @param sendQueueCapacity max batches waiting to send * @return unconnected sender */ public static QwpWebSocketSender createForTesting( String host, int port, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, int sendQueueCapacity) { + int inFlightWindowSize) { return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - inFlightWindowSize, sendQueueCapacity + inFlightWindowSize ); // Note: does NOT call ensureConnected() } @@ -730,13 +723,6 @@ public int getOrAddGlobalSymbol(String value) { return globalId; } - /** - * Returns the send queue capacity. - */ - public int getSendQueueCapacity() { - return sendQueueCapacity; - } - /** * Gets or creates a table buffer for direct access. * For high-throughput generators that want to bypass fluent API overhead. @@ -1119,7 +1105,6 @@ private void ensureConnected() { // The send queue handles both sending AND receiving (single I/O thread) if (inFlightWindowSize > 1) { sendQueue = new WebSocketSendQueue(client, inFlightWindow, - sendQueueCapacity, WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index 9a8a8bb..f567d38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -42,29 +42,29 @@ *

    * This class manages a dedicated I/O thread that handles both: *

      - *
    • Sending batches from a bounded queue
      • + *
    • Sending batches via a single-slot handoff (volatile reference)
      • *
    • Receiving and processing server ACK responses
      • *
    + * The single-slot design matches the double-buffering scheme: at most one + * sealed buffer is pending while the other is being filled. * Using a single thread eliminates concurrency issues with the WebSocket channel. *

    * Thread safety: *

      - *
    • The send queue is thread-safe for concurrent access
      • + *
    • The pending slot is thread-safe for concurrent access
      • *
    • Only the I/O thread interacts with the WebSocket channel
      • *
    • Buffer state transitions ensure safe hand-over
      • *
    *

    * Backpressure: *

      - *
    • When the queue is full, {@link #enqueue} blocks
      • + *
    • When the slot is occupied, {@link #enqueue} blocks
      • *
    • This propagates backpressure to the user thread
      • *
    */ public class WebSocketSendQueue implements QuietCloseable { public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; - // Default configuration - public static final int DEFAULT_QUEUE_CAPACITY = 16; public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); // The WebSocket client for I/O (single-threaded access only) @@ -113,7 +113,7 @@ public class WebSocketSendQueue implements QuietCloseable { * @param client the WebSocket client for I/O */ public WebSocketSendQueue(WebSocketClient client) { - this(client, null, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + this(client, null, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); } /** @@ -123,7 +123,7 @@ public WebSocketSendQueue(WebSocketClient client) { * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) */ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow) { - this(client, inFlightWindow, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + this(client, inFlightWindow, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); } /** @@ -131,18 +131,14 @@ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFli * * @param client the WebSocket client for I/O * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) - * @param queueCapacity maximum number of pending batches * @param enqueueTimeoutMs timeout for enqueue operations (ms) * @param shutdownTimeoutMs timeout for graceful shutdown (ms) */ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow, - int queueCapacity, long enqueueTimeoutMs, long shutdownTimeoutMs) { + long enqueueTimeoutMs, long shutdownTimeoutMs) { if (client == null) { throw new IllegalArgumentException("client cannot be null"); } - if (queueCapacity <= 0) { - throw new IllegalArgumentException("queueCapacity must be positive"); - } this.client = client; this.inFlightWindow = inFlightWindow; @@ -157,7 +153,7 @@ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFli this.ioThread.setDaemon(true); this.ioThread.start(); - LOG.info("WebSocket I/O thread started [capacity={}]", queueCapacity); + LOG.info("WebSocket I/O thread started"); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 88715be..21f47b0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -73,7 +73,6 @@ public class QwpAllocationTestClient { private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8) private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; private static final int DEFAULT_ROWS = 80_000_000; - private static final int DEFAULT_SEND_QUEUE = 0; // 0 = use protocol default (16) private static final int DEFAULT_WARMUP_ROWS = 100_000; private static final String PROTOCOL_ILP_HTTP = "ilp-http"; // Protocol modes @@ -100,7 +99,6 @@ public static void main(String[] args) { int flushBytes = DEFAULT_FLUSH_BYTES; long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS; int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW; - int sendQueue = DEFAULT_SEND_QUEUE; int warmupRows = DEFAULT_WARMUP_ROWS; int reportInterval = DEFAULT_REPORT_INTERVAL; @@ -124,8 +122,6 @@ public static void main(String[] args) { flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length())); } else if (arg.startsWith("--in-flight-window=")) { inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length())); - } else if (arg.startsWith("--send-queue=")) { - sendQueue = Integer.parseInt(arg.substring("--send-queue=".length())); } else if (arg.startsWith("--warmup=")) { warmupRows = Integer.parseInt(arg.substring("--warmup=".length())); } else if (arg.startsWith("--report=")) { @@ -157,14 +153,13 @@ public static void main(String[] args) { System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes))); System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms")); System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow)); - System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue)); System.out.println("Warmup rows: " + String.format("%,d", warmupRows)); System.out.println("Report interval: " + String.format("%,d", reportInterval)); System.out.println(); try { runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue, warmupRows, reportInterval); + inFlightWindow, warmupRows, reportInterval); } catch (Exception e) { System.err.println("Error: " + e.getMessage()); e.printStackTrace(System.err); @@ -174,7 +169,7 @@ public static void main(String[] args) { private static Sender createSender(String protocol, String host, int port, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { + int inFlightWindow) { switch (protocol) { case PROTOCOL_ILP_TCP: return Sender.builder(Sender.Transport.TCP) @@ -196,7 +191,6 @@ private static Sender createSender(String protocol, String host, int port, if (flushBytes > 0) b.autoFlushBytes(flushBytes); if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); return b.build(); default: throw new IllegalArgumentException("Unknown protocol: " + protocol + @@ -260,12 +254,12 @@ private static void printUsage() { private static void runTest(String protocol, String host, int port, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue, + int inFlightWindow, int warmupRows, int reportInterval) throws IOException { System.out.println("Connecting to " + host + ":" + port + "..."); try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue)) { + inFlightWindow)) { System.out.println("Connected! Protocol: " + protocol); System.out.println(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index bed59a3..289643c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -68,7 +68,6 @@ public class StacBenchmarkClient { private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; private static final int DEFAULT_ROWS = 80_000_000; - private static final int DEFAULT_SEND_QUEUE = 0; private static final String DEFAULT_TABLE = "q"; private static final int DEFAULT_WARMUP_ROWS = 100_000; // Estimated row size for throughput calculation: @@ -103,7 +102,6 @@ public static void main(String[] args) { int flushBytes = DEFAULT_FLUSH_BYTES; long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS; int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW; - int sendQueue = DEFAULT_SEND_QUEUE; int warmupRows = DEFAULT_WARMUP_ROWS; int reportInterval = DEFAULT_REPORT_INTERVAL; String table = DEFAULT_TABLE; @@ -128,8 +126,6 @@ public static void main(String[] args) { flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length())); } else if (arg.startsWith("--in-flight-window=")) { inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length())); - } else if (arg.startsWith("--send-queue=")) { - sendQueue = Integer.parseInt(arg.substring("--send-queue=".length())); } else if (arg.startsWith("--warmup=")) { warmupRows = Integer.parseInt(arg.substring("--warmup=".length())); } else if (arg.startsWith("--report=")) { @@ -160,7 +156,6 @@ public static void main(String[] args) { System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes))); System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms")); System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow)); - System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue)); System.out.println("Warmup rows: " + String.format("%,d", warmupRows)); System.out.println("Report interval: " + String.format("%,d", reportInterval)); System.out.println("Symbols: " + String.format("%,d", SYMBOL_COUNT) + " unique 4-letter tickers"); @@ -168,7 +163,7 @@ public static void main(String[] args) { try { runTest(protocol, host, port, table, totalRows, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue, warmupRows, reportInterval); + inFlightWindow, warmupRows, reportInterval); } catch (Exception e) { System.err.println("Error: " + e.getMessage()); e.printStackTrace(); @@ -178,7 +173,7 @@ public static void main(String[] args) { private static Sender createSender(String protocol, String host, int port, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { + int inFlightWindow) { switch (protocol) { case PROTOCOL_ILP_TCP: return Sender.builder(Sender.Transport.TCP) @@ -200,7 +195,6 @@ private static Sender createSender(String protocol, String host, int port, if (flushBytes > 0) b.autoFlushBytes(flushBytes); if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); return b.build(); default: throw new IllegalArgumentException("Unknown protocol: " + protocol + @@ -288,12 +282,12 @@ private static void printUsage() { private static void runTest(String protocol, String host, int port, String table, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue, + int inFlightWindow, int warmupRows, int reportInterval) throws IOException { System.out.println("Connecting to " + host + ":" + port + "..."); try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue)) { + inFlightWindow)) { System.out.println("Connected! Protocol: " + protocol); System.out.println(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index ecb2bb7..0eeccd1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -107,8 +107,7 @@ public void testAsyncModeWithAllOptions() { .autoFlushRows(500) .autoFlushBytes(512 * 1024) .autoFlushIntervalMillis(50) - .inFlightWindowSize(8) - .sendQueueCapacity(16); + .inFlightWindowSize(8); Assert.assertNotNull(builder); } @@ -319,8 +318,7 @@ public void testFullAsyncConfiguration() { .autoFlushRows(1000) .autoFlushBytes(1024 * 1024) .autoFlushIntervalMillis(100) - .inFlightWindowSize(16) - .sendQueueCapacity(32); + .inFlightWindowSize(16); Assert.assertNotNull(builder); } @@ -333,8 +331,7 @@ public void testFullAsyncConfigurationWithTls() { .asyncMode(true) .autoFlushRows(1000) .autoFlushBytes(1024 * 1024) - .inFlightWindowSize(16) - .sendQueueCapacity(32); + .inFlightWindowSize(16); Assert.assertNotNull(builder); } @@ -524,52 +521,6 @@ public void testRetryTimeout_mayNotApply() { Assert.assertNotNull(builder); } - @Test - public void testSendQueueCapacityDoubleSet_fails() { - assertThrows("already configured", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(16) - .sendQueueCapacity(32)); - } - - @Test - public void testSendQueueCapacityNegative_fails() { - assertThrows("must be positive", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(-1)); - } - - @Test - public void testSendQueueCapacityZero_fails() { - assertThrows("must be positive", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(0)); - } - - @Test - public void testSendQueueCapacity_withAsyncMode() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(32); - Assert.assertNotNull(builder); - } - - @Test - public void testSendQueueCapacity_withoutAsyncMode_fails() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .sendQueueCapacity(32), - "requires async mode"); - } - @Test public void testSyncModeAutoFlushDefaults() throws Exception { // Regression test: sync-mode connect() must not hardcode autoFlush to 0. @@ -606,16 +557,6 @@ public void testSyncModeDoesNotAllowInFlightWindowSize() { "requires async mode"); } - @Test - public void testSyncModeDoesNotAllowSendQueueCapacity() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false) - .sendQueueCapacity(32), - "requires async mode"); - } - @Test public void testSyncModeIsDefault() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index f67ef61..4be668c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -49,7 +49,7 @@ public class QwpWebSocketSenderStateTest extends AbstractTest { public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { TestUtils.assertMemoryLeak(() -> { QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 1, 10_000_000, 0, 1, 16 + "localhost", 0, 1, 10_000_000, 0, 1 ); try { setField(sender, "connected", true); @@ -91,7 +91,7 @@ public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Exception { TestUtils.assertMemoryLeak(() -> { QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 1, 10_000_000, 0, 1, 16 + "localhost", 0, 1, 10_000_000, 0, 1 ); try { setField(sender, "connected", true); @@ -127,7 +127,7 @@ public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception TestUtils.assertMemoryLeak(() -> { // Use high autoFlushRows to prevent auto-flush during the test QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 10_000, 10_000_000, 0, 1, 16 + "localhost", 0, 10_000, 10_000_000, 0, 1 ); try { // Bypass ensureConnected() — mark as connected, leave client null From c58b7cd90ef9bd93aeff05da4ea4d720c42d6d25 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 10:10:34 +0100 Subject: [PATCH 063/230] Clarify wire vs buffer type size javadoc Rename elementSize() to elementSizeInBuffer() in QwpTableBuffer to make explicit that it returns the in-memory buffer stride, not the wire-format encoding size. Update javadoc on both elementSizeInBuffer() and QwpConstants.getFixedTypeSize() to document the distinction: getFixedTypeSize() returns wire sizes (0 for bit-packed BOOLEAN, -1 for variable-width GEOHASH), while elementSizeInBuffer() returns the off-heap buffer stride (1 for BOOLEAN, 8 for GEOHASH). Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpConstants.java | 9 +++++++-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 54d6fdd..a6142c9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -295,10 +295,15 @@ private QwpConstants() { } /** - * Returns the size in bytes for fixed-width types. + * Returns the per-value size in bytes as encoded on the wire. BOOLEAN returns 0 + * because it is bit-packed (1 bit per value). GEOHASH returns -1 because it uses + * variable-width encoding (varint precision + ceil(precision/8) bytes per value). + *

    + * This is distinct from the in-memory buffer stride used by the client's + * {@code QwpTableBuffer.elementSizeInBuffer()}. * * @param typeCode the column type code (without nullable flag) - * @return size in bytes, or -1 for variable-width types + * @return size in bytes, 0 for bit-packed (BOOLEAN), or -1 for variable-width types */ public static int getFixedTypeSize(byte typeCode) { int code = typeCode & TYPE_MASK; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 119bbbc..5fd3b47 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -257,10 +257,20 @@ public void reset() { } /** - * Returns the element size in bytes for a fixed-width column type. - * Returns 0 for variable-width types (string, arrays). + * Returns the in-memory buffer element stride in bytes. This is the size used + * to store each value in the client's off-heap {@link OffHeapAppendMemory} buffer. + * This is different from element size on the wire. + *

    + * For example, BOOLEAN is stored as 1 byte per value here (for easy indexed access) + * but bit-packed on the wire; GEOHASH is stored as 8-byte longs here but uses + * variable-width encoding on the wire. + *

    + * Returns 0 for variable-width types (string, arrays) that do not use a fixed-stride + * data buffer. + * + * @see QwpConstants#getFixedTypeSize(byte) for wire-format sizes */ - static int elementSize(byte type) { + static int elementSizeInBuffer(byte type) { switch (type) { case TYPE_BOOLEAN: case TYPE_BYTE: @@ -420,7 +430,7 @@ public ColumnBuffer(String name, byte type, boolean nullable) { this.name = name; this.type = type; this.nullable = nullable; - this.elemSize = elementSize(type); + this.elemSize = elementSizeInBuffer(type); this.size = 0; this.valueCount = 0; this.hasNulls = false; From 408c88967219a12bf7cb94684aca9ae0b389cc49 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 11:07:36 +0100 Subject: [PATCH 064/230] Round out GEOHASH support --- .../qwp/client/QwpWebSocketEncoder.java | 30 +++++++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 79 +++++++++++-------- 2 files changed, 75 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 0ec22b2..9ee86b5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -203,6 +203,9 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, case TYPE_DATE: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; + case TYPE_GEOHASH: + writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); + break; case TYPE_STRING: case TYPE_VARCHAR: writeStringColumn(col, valueCount); @@ -280,6 +283,9 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC case TYPE_DATE: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; + case TYPE_GEOHASH: + writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); + break; case TYPE_STRING: case TYPE_VARCHAR: writeStringColumn(col, valueCount); @@ -416,6 +422,30 @@ private void writeDecimal64Column(byte scale, long addr, int count) { } } + /** + * Writes a GeoHash column in variable-width wire format. + *

    + * Wire format: [precision varint] [packed values: ceil(precision/8) bytes each] + * Values are stored as 8-byte longs in the off-heap buffer but only the + * lower ceil(precision/8) bytes are written to the wire. + */ + private void writeGeoHashColumn(long addr, int count, int precision) { + if (precision < 1) { + // All values are null: use minimum valid precision. + // The decoder will skip all values via the null bitmap, + // so the precision only needs to be structurally valid. + precision = 1; + } + buffer.putVarint(precision); + int valueSize = (precision + 7) / 8; + for (int i = 0; i < count; i++) { + long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8); + for (int b = 0; b < valueSize; b++) { + buffer.putByte((byte) (value >>> (b * 8))); + } + } + } + private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 5fd3b47..b4dc34a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -411,6 +411,8 @@ public static class ColumnBuffer implements QuietCloseable { // Decimal storage private byte decimalScale = -1; private double[] doubleArrayData; + // GeoHash precision (number of bits, 1-60) + private int geohashPrecision = -1; private boolean hasNulls; private long[] longArrayData; private int maxGlobalSymbolId = -1; @@ -632,6 +634,29 @@ public void addFloat(float value) { size++; } + /** + * Adds a geohash value with the given precision. + * + * @param value the geohash value (bit-packed) + * @param precision number of bits (1-60) + */ + public void addGeoHash(long value, int precision) { + if (precision < 1 || precision > 60) { + throw new LineSenderException("invalid GeoHash precision: " + precision + " (must be 1-60)"); + } + if (geohashPrecision == -1) { + geohashPrecision = precision; + } else if (geohashPrecision != precision) { + throw new LineSenderException( + "GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision + ); + } + ensureNullBitmapForNonNull(); + dataBuffer.putLong(value); + valueCount++; + size++; + } + public void addInt(int value) { ensureNullBitmapForNonNull(); dataBuffer.putInt(value); @@ -754,6 +779,13 @@ public void addNull() { if (nullable) { ensureNullCapacity(size + 1); markNull(size); + // GEOHASH uses dense wire format: all rows (including nulls) + // occupy space in the values array. Write a placeholder value + // so the data buffer stays aligned with the row index. + if (type == TYPE_GEOHASH) { + dataBuffer.putLong(0L); + valueCount++; + } size++; } else { // For non-nullable columns, store a sentinel/default value @@ -928,18 +960,10 @@ public void close() { } } - public int getArrayDataOffset() { - return arrayDataOffset; - } - public byte[] getArrayDims() { return arrayDims; } - public int getArrayShapeOffset() { - return arrayShapeOffset; - } - public int[] getArrayShapes() { return arrayShapes; } @@ -974,6 +998,10 @@ public double[] getDoubleArrayData() { return doubleArrayData; } + public int getGeoHashPrecision() { + return geohashPrecision; + } + public long[] getLongArrayData() { return longArrayData; } @@ -1021,10 +1049,6 @@ public String[] getSymbolDictionary() { return dict; } - public int getSymbolDictionarySize() { - return symbolList == null ? 0 : symbolList.size(); - } - public byte getType() { return type; } @@ -1033,10 +1057,6 @@ public int getValueCount() { return valueCount; } - public boolean hasNulls() { - return hasNulls; - } - public boolean isNull(int index) { if (nullBufPtr == 0) { return false; @@ -1074,6 +1094,7 @@ public void reset() { arrayShapeOffset = 0; arrayDataOffset = 0; decimalScale = -1; + geohashPrecision = -1; } public void truncateTo(int newSize) { @@ -1153,6 +1174,7 @@ private void allocateStorage(byte type) { dataBuffer = new OffHeapAppendMemory(32); break; case TYPE_INT: + case TYPE_FLOAT: dataBuffer = new OffHeapAppendMemory(64); break; case TYPE_GEOHASH: @@ -1160,13 +1182,16 @@ private void allocateStorage(byte type) { case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: case TYPE_DATE: + case TYPE_DECIMAL64: + case TYPE_DOUBLE: dataBuffer = new OffHeapAppendMemory(128); break; - case TYPE_FLOAT: - dataBuffer = new OffHeapAppendMemory(64); + case TYPE_DECIMAL128: + dataBuffer = new OffHeapAppendMemory(256); break; - case TYPE_DOUBLE: - dataBuffer = new OffHeapAppendMemory(128); + case TYPE_LONG256: + case TYPE_DECIMAL256: + dataBuffer = new OffHeapAppendMemory(512); break; case TYPE_STRING: case TYPE_VARCHAR: @@ -1180,25 +1205,11 @@ private void allocateStorage(byte type) { symbolList = new ObjList<>(); break; case TYPE_UUID: - dataBuffer = new OffHeapAppendMemory(256); - break; - case TYPE_LONG256: - dataBuffer = new OffHeapAppendMemory(512); - break; case TYPE_DOUBLE_ARRAY: case TYPE_LONG_ARRAY: arrayDims = new byte[16]; arrayCapture = new ArrayCapture(); break; - case TYPE_DECIMAL64: - dataBuffer = new OffHeapAppendMemory(128); - break; - case TYPE_DECIMAL128: - dataBuffer = new OffHeapAppendMemory(256); - break; - case TYPE_DECIMAL256: - dataBuffer = new OffHeapAppendMemory(512); - break; } } From 11a416c8dc1e2abc37ea983c6d5dc9d591e822d6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 12:43:04 +0100 Subject: [PATCH 065/230] Deduplicate column encoding in QwpWebSocketEncoder encodeColumn() and encodeColumnWithGlobalSymbols() had nearly identical switch statements across 15+ type cases, differing only in the SYMBOL case. Merge them into a single encodeColumn() with a boolean useGlobalSymbols parameter. Similarly merge the duplicate encodeTable() and encodeTableWithGlobalSymbols() into one method. This removes ~100 lines of duplicated code. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketEncoder.java | 115 ++---------------- 1 file changed, 10 insertions(+), 105 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 9ee86b5..c62193d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -79,7 +79,7 @@ public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { buffer.reset(); writeHeader(1, 0); int payloadStart = buffer.getPosition(); - encodeTable(tableBuffer, useSchemaRef); + encodeTable(tableBuffer, useSchemaRef, false); int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); return buffer.getPosition(); @@ -105,7 +105,7 @@ public int encodeWithDeltaDict( String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } - encodeTableWithGlobalSymbols(tableBuffer, useSchemaRef); + encodeTable(tableBuffer, useSchemaRef, true); int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); flags = savedFlags; @@ -165,7 +165,7 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putInt(payloadLength); } - private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { + private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla, boolean useGlobalSymbols) { int valueCount = col.getValueCount(); long dataAddr = col.getDataAddress(); @@ -211,7 +211,11 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, writeStringColumn(col, valueCount); break; case TYPE_SYMBOL: - writeSymbolColumn(col, valueCount); + if (useGlobalSymbols) { + writeSymbolColumnWithGlobalIds(col, valueCount); + } else { + writeSymbolColumn(col, valueCount); + } break; case TYPE_UUID: // Stored as lo+hi contiguously, matching wire order @@ -241,106 +245,7 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, } } - private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { - int valueCount = col.getValueCount(); - - if (colDef.isNullable()) { - writeNullBitmap(col, rowCount); - } - - if (col.getType() == TYPE_SYMBOL) { - writeSymbolColumnWithGlobalIds(col, valueCount); - } else { - // Delegate to standard encoding for all other types - long dataAddr = col.getDataAddress(); - switch (col.getType()) { - case TYPE_BOOLEAN: - writeBooleanColumn(dataAddr, valueCount); - break; - case TYPE_BYTE: - buffer.putBlockOfBytes(dataAddr, valueCount); - break; - case TYPE_SHORT: - case TYPE_CHAR: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); - break; - case TYPE_INT: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); - break; - case TYPE_LONG: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_FLOAT: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); - break; - case TYPE_DOUBLE: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - writeTimestampColumn(dataAddr, valueCount, useGorilla); - break; - case TYPE_DATE: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_GEOHASH: - writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); - break; - case TYPE_STRING: - case TYPE_VARCHAR: - writeStringColumn(col, valueCount); - break; - case TYPE_UUID: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); - break; - case TYPE_LONG256: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); - break; - case TYPE_DOUBLE_ARRAY: - writeDoubleArrayColumn(col, valueCount); - break; - case TYPE_LONG_ARRAY: - writeLongArrayColumn(col, valueCount); - break; - case TYPE_DECIMAL64: - writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); - break; - case TYPE_DECIMAL128: - writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); - break; - case TYPE_DECIMAL256: - writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); - break; - default: - throw new LineSenderException("Unknown column type: " + col.getType()); - } - } - } - - private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { - QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - int rowCount = tableBuffer.getRowCount(); - - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - tableBuffer.getSchemaHash(), - columnDefs.length - ); - } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); - } - - boolean useGorilla = isGorillaEnabled(); - for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - QwpColumnDef colDef = columnDefs[i]; - encodeColumn(col, colDef, rowCount, useGorilla); - } - } - - private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols) { QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); @@ -359,7 +264,7 @@ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean us for (int i = 0; i < tableBuffer.getColumnCount(); i++) { QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); QwpColumnDef colDef = columnDefs[i]; - encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + encodeColumn(col, colDef, rowCount, useGorilla, useGlobalSymbols); } } From bde1a51913343fbb4123a867564a88cc7e7ef08a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 12:46:34 +0100 Subject: [PATCH 066/230] Delete WebSocketChannel and ResponseReader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove WebSocketChannel, ResponseReader, and WebSocketChannelTest. These classes are dead code — the actual sender implementation (QwpWebSocketSender) uses WebSocketClient and WebSocketSendQueue instead. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/ResponseReader.java | 226 ------ .../cutlass/qwp/client/WebSocketChannel.java | 661 ------------------ .../qwp/client/WebSocketChannelTest.java | 476 ------------- 3 files changed, 1363 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java deleted file mode 100644 index 552e372..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java +++ /dev/null @@ -1,226 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.client; - -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.std.QuietCloseable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Reads server responses from WebSocket channel and updates InFlightWindow. - *

    - * This class runs a dedicated thread that: - *

      - *
    • Reads WebSocket frames from the server
      • - *
    • Parses binary responses containing ACK/error status
      • - *
    • Updates the InFlightWindow with acknowledgments or failures
      • - *
    - *

    - * Thread safety: This class is thread-safe. The reader thread processes - * responses independently of the sender thread. - */ -public class ResponseReader implements QuietCloseable { - - private static final int DEFAULT_READ_TIMEOUT_MS = 100; - private static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; - private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); - private final WebSocketChannel channel; - private final InFlightWindow inFlightWindow; - private final Thread readerThread; - private final WebSocketResponse response; - private final CountDownLatch shutdownLatch; - // Statistics - private final AtomicLong totalAcks = new AtomicLong(0); - private final AtomicLong totalErrors = new AtomicLong(0); - private volatile Throwable lastError; - // State - private volatile boolean running; - - /** - * Creates a new response reader. - * - * @param channel the WebSocket channel to read from - * @param inFlightWindow the window to update with acknowledgments - */ - public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) { - if (channel == null) { - throw new IllegalArgumentException("channel cannot be null"); - } - if (inFlightWindow == null) { - throw new IllegalArgumentException("inFlightWindow cannot be null"); - } - - this.channel = channel; - this.inFlightWindow = inFlightWindow; - this.response = new WebSocketResponse(); - - this.running = true; - this.shutdownLatch = new CountDownLatch(1); - - // Start reader thread - this.readerThread = new Thread(this::readLoop, "questdb-websocket-response-reader"); - this.readerThread.setDaemon(true); - this.readerThread.start(); - - LOG.info("Response reader started"); - } - - @Override - public void close() { - if (!running) { - return; - } - - LOG.info("Closing response reader"); - - running = false; - - // Wait for reader thread to finish - try { - shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); - } - - /** - * Returns the last error that occurred, or null if no error. - */ - public Throwable getLastError() { - return lastError; - } - - /** - * Returns total successful acknowledgments received. - */ - public long getTotalAcks() { - return totalAcks.get(); - } - - /** - * Returns total error responses received. - */ - public long getTotalErrors() { - return totalErrors.get(); - } - - /** - * Returns true if the reader is still running. - */ - public boolean isRunning() { - return running; - } - - /** - * Main read loop that processes incoming WebSocket frames. - */ - private void readLoop() { - LOG.info("Read loop started"); - - try { - while (running && channel.isConnected()) { - try { - // Non-blocking read with short timeout - boolean received = channel.receiveFrame(new ResponseHandlerImpl(), DEFAULT_READ_TIMEOUT_MS); - if (!received) { - // No frame available, continue polling - continue; - } - } catch (LineSenderException e) { - if (running) { - LOG.error("Error reading response: {}", e.getMessage()); - lastError = e; - } - // Continue trying to read unless we're shutting down - } catch (Throwable t) { - if (running) { - LOG.error("Unexpected error in read loop: {}", String.valueOf(t)); - lastError = t; - } - break; - } - } - } finally { - shutdownLatch.countDown(); - LOG.info("Read loop stopped"); - } - } - - /** - * Handler for received WebSocket frames. - */ - private class ResponseHandlerImpl implements WebSocketChannel.ResponseHandler { - - @Override - public void onBinaryMessage(long payload, int length) { - if (length < WebSocketResponse.MIN_RESPONSE_SIZE) { - LOG.error("Response too short [length={}]", length); - return; - } - - // Parse response from binary payload - if (!response.readFrom(payload, length)) { - LOG.error("Failed to parse response"); - return; - } - - long sequence = response.getSequence(); - - if (response.isSuccess()) { - // Cumulative ACK - acknowledge all batches up to this sequence - int acked = inFlightWindow.acknowledgeUpTo(sequence); - if (acked > 0) { - totalAcks.addAndGet(acked); - LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); - } else { - LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); - } - } else { - // Error - fail the batch - String errorMessage = response.getErrorMessage(); - LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage); - - LineSenderException error = new LineSenderException( - "Server error for batch " + sequence + ": " + - response.getStatusName() + " - " + errorMessage); - inFlightWindow.fail(sequence, error); - totalErrors.incrementAndGet(); - } - } - - @Override - public void onClose(int code, String reason) { - LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason); - running = false; - } - } -} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java deleted file mode 100644 index d3965fa..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java +++ /dev/null @@ -1,661 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.client; - -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; -import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; -import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; -import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.SecureRnd; -import io.questdb.client.std.Unsafe; - -import javax.net.SocketFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.nio.charset.StandardCharsets; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; -import java.util.Base64; - -/** - * WebSocket client channel for ILP v4 binary streaming. - *

    - * This class handles: - *

      - *
    • HTTP upgrade handshake to establish WebSocket connection
      • - *
    • Binary frame encoding with client-side masking (RFC 6455)
      • - *
    • Ping/pong for connection keepalive
      • - *
    • Close handshake
      • - *
    - *

    - * Thread safety: This class is NOT thread-safe. It should only be accessed - * from a single thread at a time. - */ -public class WebSocketChannel implements QuietCloseable { - - private static final int DEFAULT_BUFFER_SIZE = 65536; - private static final int MAX_FRAME_HEADER_SIZE = 14; // 2 + 8 + 4 (header + extended len + mask) - // Frame parser (reused) - private final WebSocketFrameParser frameParser; - // Temporary byte array for handshake (allocated once) - private final byte[] handshakeBuffer = new byte[4096]; - // Connection state - private final String host; - private final String path; - private final int port; - // Random for mask key generation (ChaCha20-based CSPRNG, RFC 6455 Section 5.3) - private final SecureRnd rnd; - private final boolean tlsEnabled; - private final boolean tlsValidationEnabled; - private boolean closed; - // Timeouts - private int connectTimeoutMs = 10_000; - // State - private boolean connected; - private InputStream in; - private OutputStream out; - private byte[] readTempBuffer; - private int readTimeoutMs = 30_000; - private int recvBufferPos; // Write position - // Pre-allocated receive buffer (native memory) - private long recvBufferPtr; - private int recvBufferReadPos; // Read position - private int recvBufferSize; - // Pre-allocated send buffer (native memory) - private long sendBufferPtr; - private int sendBufferSize; - // Socket I/O - private Socket socket; - // Separate temp buffers for read and write to avoid race conditions - // between send queue thread and response reader thread - private byte[] writeTempBuffer; - - public WebSocketChannel(String url, boolean tlsEnabled) { - this(url, tlsEnabled, true); - } - - public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEnabled) { - // Parse URL: ws://host:port/path or wss://host:port/path - String remaining = url; - if (remaining.startsWith("wss://")) { - remaining = remaining.substring(6); - this.tlsEnabled = true; - } else if (remaining.startsWith("ws://")) { - remaining = remaining.substring(5); - this.tlsEnabled = tlsEnabled; - } else { - this.tlsEnabled = tlsEnabled; - } - - int slashIdx = remaining.indexOf('/'); - String hostPort; - if (slashIdx >= 0) { - hostPort = remaining.substring(0, slashIdx); - this.path = remaining.substring(slashIdx); - } else { - hostPort = remaining; - this.path = "/"; - } - - int colonIdx = hostPort.lastIndexOf(':'); - if (colonIdx >= 0) { - this.host = hostPort.substring(0, colonIdx); - this.port = Integer.parseInt(hostPort.substring(colonIdx + 1)); - } else { - this.host = hostPort; - this.port = this.tlsEnabled ? 443 : 80; - } - - this.tlsValidationEnabled = tlsValidationEnabled; - this.frameParser = new WebSocketFrameParser(); - this.rnd = new SecureRnd(); - - // Allocate native buffers - this.sendBufferSize = DEFAULT_BUFFER_SIZE; - this.sendBufferPtr = Unsafe.malloc(sendBufferSize, MemoryTag.NATIVE_DEFAULT); - - this.recvBufferSize = DEFAULT_BUFFER_SIZE; - this.recvBufferPtr = Unsafe.malloc(recvBufferSize, MemoryTag.NATIVE_DEFAULT); - this.recvBufferPos = 0; - this.recvBufferReadPos = 0; - - this.connected = false; - this.closed = false; - } - - /** - * Sends a close frame and closes the connection. - */ - @Override - public void close() { - if (closed) { - return; - } - closed = true; - - try { - if (connected) { - // Send close frame - sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); - } - } catch (Exception e) { - // Ignore errors during close - } - - closeQuietly(); - - // Free native memory - if (sendBufferPtr != 0) { - Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); - sendBufferPtr = 0; - } - if (recvBufferPtr != 0) { - Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); - recvBufferPtr = 0; - } - } - - /** - * Connects to the WebSocket server. - * Performs TCP connection and HTTP upgrade handshake. - */ - public void connect() { - if (connected) { - return; - } - if (closed) { - throw new LineSenderException("WebSocket channel is closed"); - } - - try { - // Create socket - SocketFactory socketFactory = tlsEnabled ? createSslSocketFactory() : SocketFactory.getDefault(); - socket = socketFactory.createSocket(); - socket.connect(new java.net.InetSocketAddress(host, port), connectTimeoutMs); - socket.setSoTimeout(readTimeoutMs); - socket.setTcpNoDelay(true); - - in = socket.getInputStream(); - out = socket.getOutputStream(); - - // Perform WebSocket handshake - performHandshake(); - - connected = true; - } catch (IOException e) { - closeQuietly(); - throw new LineSenderException("Failed to connect to WebSocket server: " + e.getMessage(), e); - } - } - - public boolean isConnected() { - return connected && !closed; - } - - /** - * Receives and processes incoming frames. - * Handles ping/pong automatically. - * - * @param handler callback for received binary messages (may be null) - * @param timeoutMs read timeout in milliseconds - * @return true if a frame was received, false on timeout - */ - public boolean receiveFrame(ResponseHandler handler, int timeoutMs) { - ensureConnected(); - try { - int oldTimeout = socket.getSoTimeout(); - socket.setSoTimeout(timeoutMs); - try { - return doReceiveFrame(handler); - } finally { - socket.setSoTimeout(oldTimeout); - } - } catch (SocketTimeoutException e) { - return false; - } catch (IOException e) { - throw new LineSenderException("Failed to receive WebSocket frame: " + e.getMessage(), e); - } - } - - /** - * Sends binary data as a WebSocket binary frame. - * The data is read from native memory at the given pointer. - * - * @param dataPtr pointer to the data - * @param length length of data in bytes - */ - public void sendBinary(long dataPtr, int length) { - ensureConnected(); - sendFrame(WebSocketOpcode.BINARY, dataPtr, length); - } - - /** - * Sends a ping frame. - */ - public void sendPing() { - ensureConnected(); - sendFrame(WebSocketOpcode.PING, 0, 0); - } - - /** - * Sets the connection timeout. - */ - public WebSocketChannel setConnectTimeout(int timeoutMs) { - this.connectTimeoutMs = timeoutMs; - return this; - } - - /** - * Sets the read timeout. - */ - public WebSocketChannel setReadTimeout(int timeoutMs) { - this.readTimeoutMs = timeoutMs; - return this; - } - - private void closeQuietly() { - connected = false; - if (socket != null) { - try { - socket.close(); - } catch (IOException e) { - // Ignore - } - socket = null; - } - in = null; - out = null; - } - - private SocketFactory createSslSocketFactory() { - try { - if (!tlsValidationEnabled) { - SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, new TrustManager[]{new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] certs, String t) { - } - - public void checkServerTrusted(X509Certificate[] certs, String t) { - } - - public X509Certificate[] getAcceptedIssuers() { - return null; - } - }}, new SecureRandom()); - return sslContext.getSocketFactory(); - } - return SSLSocketFactory.getDefault(); - } catch (Exception e) { - throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); - } - } - - private boolean doReceiveFrame(ResponseHandler handler) throws IOException { - // First, try to parse any data already in the buffer - // This handles the case where multiple frames arrived in a single TCP read - if (recvBufferPos > recvBufferReadPos) { - Boolean result = tryParseFrame(handler); - if (result != null) { - return result; - } - // result == null means we need more data, continue to read - } - - // Read more data into receive buffer - int bytesRead = readFromSocket(); - if (bytesRead <= 0) { - return false; - } - - // Try parsing again with the new data - Boolean result = tryParseFrame(handler); - return result != null && result; - } - - private void ensureConnected() { - if (closed) { - throw new LineSenderException("WebSocket channel is closed"); - } - if (!connected) { - throw new LineSenderException("WebSocket channel is not connected"); - } - } - - private void ensureSendBufferSize(int required) { - if (required > sendBufferSize) { - int newSize = Math.max(required, sendBufferSize * 2); - sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - sendBufferSize = newSize; - } - } - - private byte[] getReadTempBuffer(int minSize) { - if (readTempBuffer == null || readTempBuffer.length < minSize) { - readTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return readTempBuffer; - } - - private byte[] getWriteTempBuffer(int minSize) { - if (writeTempBuffer == null || writeTempBuffer.length < minSize) { - writeTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return writeTempBuffer; - } - - private void performHandshake() throws IOException { - // Generate random key (16 bytes, base64 encoded = 24 chars) - byte[] keyBytes = new byte[16]; - for (int i = 0; i < 16; i++) { - keyBytes[i] = (byte) rnd.nextInt(); - } - String key = Base64.getEncoder().encodeToString(keyBytes); - - // Build HTTP upgrade request - StringBuilder request = new StringBuilder(); - request.append("GET ").append(path).append(" HTTP/1.1\r\n"); - request.append("Host: ").append(host); - if ((tlsEnabled && port != 443) || (!tlsEnabled && port != 80)) { - request.append(":").append(port); - } - request.append("\r\n"); - request.append("Upgrade: websocket\r\n"); - request.append("Connection: Upgrade\r\n"); - request.append("Sec-WebSocket-Key: ").append(key).append("\r\n"); - request.append("Sec-WebSocket-Version: 13\r\n"); - request.append("\r\n"); - - // Send request - byte[] requestBytes = request.toString().getBytes(StandardCharsets.US_ASCII); - out.write(requestBytes); - out.flush(); - - // Read response - int responseLen = readHttpResponse(); - - // Parse response - String response = new String(handshakeBuffer, 0, responseLen, StandardCharsets.US_ASCII); - - // Check status line - if (!response.startsWith("HTTP/1.1 101")) { - throw new IOException("WebSocket handshake failed: " + response.split("\r\n")[0]); - } - - // Verify Sec-WebSocket-Accept - String expectedAccept = WebSocketHandshake.computeAcceptKey(key); - if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) { - throw new IOException("Invalid Sec-WebSocket-Accept in handshake response"); - } - } - - private int readFromSocket() throws IOException { - // Ensure space in receive buffer - int available = recvBufferSize - recvBufferPos; - if (available < 1024) { - // Grow buffer - int newSize = recvBufferSize * 2; - recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - recvBufferSize = newSize; - available = recvBufferSize - recvBufferPos; - } - - // Read into temp array then copy to native buffer - // Use separate read buffer to avoid race with write thread - byte[] temp = getReadTempBuffer(available); - int bytesRead = in.read(temp, 0, available); - if (bytesRead > 0) { - Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead); - recvBufferPos += bytesRead; - } - return bytesRead; - } - - private int readHttpResponse() throws IOException { - int pos = 0; - int consecutiveCrLf = 0; - - while (pos < handshakeBuffer.length) { - int b = in.read(); - if (b < 0) { - throw new IOException("Connection closed during handshake"); - } - handshakeBuffer[pos++] = (byte) b; - - // Look for \r\n\r\n - if (b == '\r' || b == '\n') { - if ((consecutiveCrLf == 0 && b == '\r') || - (consecutiveCrLf == 1 && b == '\n') || - (consecutiveCrLf == 2 && b == '\r') || - (consecutiveCrLf == 3 && b == '\n')) { - consecutiveCrLf++; - if (consecutiveCrLf == 4) { - return pos; - } - } else { - consecutiveCrLf = (b == '\r') ? 1 : 0; - } - } else { - consecutiveCrLf = 0; - } - } - throw new IOException("HTTP response too large"); - } - - private void sendCloseFrame(int code, String reason) { - int maskKey = rnd.nextInt(); - - // Close payload: 2-byte code + optional reason - // Compute UTF-8 bytes upfront so payload length is correct - byte[] reasonBytes = (reason != null) ? reason.getBytes(StandardCharsets.UTF_8) : null; - int reasonLen = (reasonBytes != null) ? reasonBytes.length : 0; - int payloadLen = 2 + reasonLen; - - int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); - int frameSize = headerSize + payloadLen; - - ensureSendBufferSize(frameSize); - - // Write header - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, WebSocketOpcode.CLOSE, payloadLen, maskKey); - - // Write close code (big-endian) - long payloadStart = sendBufferPtr + headerWritten; - Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF)); - Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF)); - - // Write reason if present - if (reasonBytes != null) { - for (int i = 0; i < reasonBytes.length; i++) { - Unsafe.getUnsafe().putByte(payloadStart + 2 + i, reasonBytes[i]); - } - } - - // Mask payload - WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey); - - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - // Ignore errors during close - } - } - - private void sendFrame(int opcode, long payloadPtr, int payloadLen) { - // Generate mask key - int maskKey = rnd.nextInt(); - - // Calculate required buffer size - int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); - int frameSize = headerSize + payloadLen; - - // Ensure buffer is large enough - ensureSendBufferSize(frameSize); - - // Write frame header with mask - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, opcode, payloadLen, maskKey); - - // Copy payload to buffer after header - if (payloadLen > 0) { - Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); - // Mask the payload in place - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); - } - - // Send frame - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); - } - } - - private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { - int maskKey = rnd.nextInt(); - int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); - int frameSize = headerSize + pingPayloadLen; - - ensureSendBufferSize(frameSize); - - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); - - if (pingPayloadLen > 0) { - Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); - } - - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - // Ignore pong send errors - } - } - - /** - * Tries to parse a frame from the receive buffer. - * - * @return true if frame processed, false if error, null if need more data - */ - private Boolean tryParseFrame(ResponseHandler handler) throws IOException { - frameParser.reset(); - int consumed = frameParser.parse( - recvBufferPtr + recvBufferReadPos, - recvBufferPtr + recvBufferPos); - - if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE) { - return null; // Need more data - } - - if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) { - throw new IOException("WebSocket frame parse error: " + frameParser.getErrorCode()); - } - - if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { - long payloadPtr = recvBufferPtr + recvBufferReadPos + frameParser.getHeaderSize(); - int payloadLen = (int) frameParser.getPayloadLength(); - - // Handle control frames - int opcode = frameParser.getOpcode(); - switch (opcode) { - case WebSocketOpcode.PING: - sendPongFrame(payloadPtr, payloadLen); - break; - case WebSocketOpcode.PONG: - // Ignore pong - break; - case WebSocketOpcode.CLOSE: - connected = false; - if (handler != null) { - int closeCode = 0; - if (payloadLen >= 2) { - closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8) - | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF); - } - handler.onClose(closeCode, null); - } - break; - case WebSocketOpcode.BINARY: - if (handler != null) { - handler.onBinaryMessage(payloadPtr, payloadLen); - } - break; - case WebSocketOpcode.TEXT: - // Ignore text frames for now - break; - } - - // Advance read position - recvBufferReadPos += consumed; - - // Compact buffer if needed - if (recvBufferReadPos > 0) { - int remaining = recvBufferPos - recvBufferReadPos; - if (remaining > 0) { - Unsafe.getUnsafe().copyMemory( - recvBufferPtr + recvBufferReadPos, - recvBufferPtr, - remaining); - } - recvBufferPos = remaining; - recvBufferReadPos = 0; - } - - return true; - } - - return false; - } - - private void writeToSocket(long ptr, int len) throws IOException { - // Copy to temp array for socket write (unavoidable with OutputStream) - // Use separate write buffer to avoid race with read thread - byte[] temp = getWriteTempBuffer(len); - Unsafe.getUnsafe().copyMemory(null, ptr, temp, Unsafe.BYTE_OFFSET, len); - out.write(temp, 0, len); - out.flush(); - } - - /** - * Callback interface for received WebSocket messages. - */ - public interface ResponseHandler { - void onBinaryMessage(long payload, int length); - - void onClose(int code, String reason); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java deleted file mode 100644 index 4616a7a..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java +++ /dev/null @@ -1,476 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client; - -import io.questdb.client.cutlass.qwp.client.WebSocketChannel; -import io.questdb.client.cutlass.qwp.client.WebSocketResponse; -import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; -import io.questdb.client.test.AbstractTest; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Tests for WebSocketChannel's native-heap memory copy paths. - * Exercises writeToSocket (native to heap) and readFromSocket (heap to native) - * through a local echo server. - */ -public class WebSocketChannelTest extends AbstractTest { - - @Test - public void testBinaryRoundTripAllByteValues() throws Exception { - TestUtils.assertMemoryLeak(() -> { - int len = 256; - long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < len; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i); - } - assertBinaryRoundTrip(sendPtr, len); - } finally { - Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT); - } - }); - } - - @Test - public void testBinaryRoundTripLargePayload() throws Exception { - TestUtils.assertMemoryLeak(() -> { - // Large payload that exercises bulk copyMemory across many cache lines. - // Kept under 32KB so the echo response arrives in a single TCP read - // on loopback (avoids a pre-existing bug in doReceiveFrame with - // partial frame assembly). - assertBinaryRoundTrip(30_000); - }); - } - - @Test - public void testBinaryRoundTripMediumPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096)); - } - - @Test - public void testBinaryRoundTripRepeatedFrames() throws Exception { - TestUtils.assertMemoryLeak(() -> { - int payloadLen = 1000; - int frameCount = 10; - long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); - try (EchoServer server = new EchoServer()) { - server.start(); - WebSocketChannel channel = new WebSocketChannel( - "localhost:" + server.getPort() + "/", false - ); - try { - channel.setConnectTimeout(5000); - channel.setReadTimeout(5000); - channel.connect(); - - for (int f = 0; f < frameCount; f++) { - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i + f)); - } - channel.sendBinary(sendPtr, payloadLen); - - ReceivedPayload received = new ReceivedPayload(); - boolean ok = receiveWithRetry(channel, received, 5000); - server.assertNoError(); - Assert.assertTrue("frame " + f + ": expected response", ok); - Assert.assertEquals("frame " + f + ": length", payloadLen, received.length); - - for (int i = 0; i < payloadLen; i++) { - Assert.assertEquals( - "frame " + f + " byte " + i, - (byte) (i + f), - Unsafe.getUnsafe().getByte(received.ptr + i) - ); - } - } - } finally { - channel.close(); - } - server.assertNoError(); - } finally { - Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT); - } - }); - } - - @Test - public void testBinaryRoundTripSmallPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); - } - - @Test - public void testResponseReadFromEmptyErrorClearsStaleMessage() { - // First, parse an error response WITH an error message - WebSocketResponse response = new WebSocketResponse(); - WebSocketResponse errorWithMsg = WebSocketResponse.error(42, WebSocketResponse.STATUS_PARSE_ERROR, "bad input"); - int size1 = errorWithMsg.serializedSize(); - long ptr = Unsafe.malloc(size1, MemoryTag.NATIVE_DEFAULT); - try { - errorWithMsg.writeTo(ptr); - Assert.assertTrue(response.readFrom(ptr, size1)); - Assert.assertEquals("bad input", response.getErrorMessage()); - } finally { - Unsafe.free(ptr, size1, MemoryTag.NATIVE_DEFAULT); - } - - // Now, parse an error response with an EMPTY error message (msgLen=0) - // but with a buffer larger than MIN_ERROR_RESPONSE_SIZE. This triggers - // the path where the outer if (length > offset + 2) is true, but the - // inner if (msgLen > 0) is false, leaving errorMessage stale. - int size2 = WebSocketResponse.MIN_ERROR_RESPONSE_SIZE + 1; - ptr = Unsafe.malloc(size2, MemoryTag.NATIVE_DEFAULT); - try { - int offset = 0; - Unsafe.getUnsafe().putByte(ptr + offset, WebSocketResponse.STATUS_WRITE_ERROR); - offset += 1; - Unsafe.getUnsafe().putLong(ptr + offset, 99L); - offset += 8; - Unsafe.getUnsafe().putShort(ptr + offset, (short) 0); // msgLen = 0 - - Assert.assertTrue(response.readFrom(ptr, size2)); - Assert.assertEquals(WebSocketResponse.STATUS_WRITE_ERROR, response.getStatus()); - Assert.assertEquals(99L, response.getSequence()); - Assert.assertNull("errorMessage should be null for empty error message", response.getErrorMessage()); - } finally { - Unsafe.free(ptr, size2, MemoryTag.NATIVE_DEFAULT); - } - } - - /** - * Calls receiveFrame in a loop to handle the case where doReceiveFrame - * needs multiple reads to assemble a complete frame (e.g. header and - * payload arrive in separate TCP segments). - */ - private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) { - long deadline = System.currentTimeMillis() + timeoutMs; - while (System.currentTimeMillis() < deadline) { - int remaining = (int) (deadline - System.currentTimeMillis()); - if (remaining <= 0) { - break; - } - if (channel.receiveFrame(handler, remaining)) { - return true; - } - } - return false; - } - - private void assertBinaryRoundTrip(int payloadLen) throws Exception { - long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i & 0xFF)); - } - assertBinaryRoundTrip(sendPtr, payloadLen); - } finally { - Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT); - } - } - - private void assertBinaryRoundTrip(long sendPtr, int payloadLen) throws Exception { - try (EchoServer server = new EchoServer()) { - server.start(); - WebSocketChannel channel = new WebSocketChannel( - "localhost:" + server.getPort() + "/", false - ); - try { - channel.setConnectTimeout(5000); - channel.setReadTimeout(5000); - channel.connect(); - - // Send exercises writeToSocket (native to heap via copyMemory) - channel.sendBinary(sendPtr, payloadLen); - - // Receive exercises readFromSocket (heap to native via copyMemory) - ReceivedPayload received = new ReceivedPayload(); - boolean ok = receiveWithRetry(channel, received, 5000); - - // Check server error before client assertions - server.assertNoError(); - Assert.assertTrue("expected a frame back from echo server", ok); - Assert.assertEquals("payload length mismatch", payloadLen, received.length); - - for (int i = 0; i < payloadLen; i++) { - byte expected = Unsafe.getUnsafe().getByte(sendPtr + i); - byte actual = Unsafe.getUnsafe().getByte(received.ptr + i); - Assert.assertEquals("byte mismatch at offset " + i, expected, actual); - } - } finally { - channel.close(); - } - server.assertNoError(); - } - } - - /** - * Minimal WebSocket echo server. Accepts one connection, completes the - * HTTP upgrade handshake, then echoes every binary frame back unmasked. - * All echo writes use a single byte array to avoid TCP fragmentation. - */ - private static class EchoServer implements AutoCloseable { - private static final Pattern KEY_PATTERN = - Pattern.compile("Sec-WebSocket-Key:\\s*(.+?)\\r\\n"); - private final AtomicReference error = new AtomicReference<>(); - private final ServerSocket serverSocket; - private Thread thread; - - EchoServer() throws IOException { - serverSocket = new ServerSocket(0); - } - - @Override - public void close() throws Exception { - serverSocket.close(); - if (thread != null) { - thread.join(5000); - } - } - - private void completeHandshake(InputStream in, OutputStream out) throws IOException { - byte[] buf = new byte[4096]; - int pos = 0; - - while (pos < buf.length) { - int b = in.read(); - if (b < 0) { - throw new IOException("connection closed during handshake"); - } - buf[pos++] = (byte) b; - if (pos >= 4 - && buf[pos - 4] == '\r' && buf[pos - 3] == '\n' - && buf[pos - 2] == '\r' && buf[pos - 1] == '\n') { - break; - } - } - - String request = new String(buf, 0, pos, StandardCharsets.US_ASCII); - Matcher m = KEY_PATTERN.matcher(request); - if (!m.find()) { - throw new IOException("no Sec-WebSocket-Key in request:\n" + request); - } - String clientKey = m.group(1).trim(); - String acceptKey = WebSocketHandshake.computeAcceptKey(clientKey); - - String response = "HTTP/1.1 101 Switching Protocols\r\n" - + "Upgrade: websocket\r\n" - + "Connection: Upgrade\r\n" - + "Sec-WebSocket-Accept: " + acceptKey + "\r\n" - + "\r\n"; - out.write(response.getBytes(StandardCharsets.US_ASCII)); - out.flush(); - } - - private void echoFrames(InputStream in, OutputStream out) throws IOException { - byte[] readBuf = new byte[256 * 1024]; - - while (true) { - int pos = 0; - while (pos < 2) { - int n = in.read(readBuf, pos, readBuf.length - pos); - if (n < 0) { - return; - } - pos += n; - } - - int byte0 = readBuf[0] & 0xFF; - int byte1 = readBuf[1] & 0xFF; - int opcode = byte0 & 0x0F; - boolean masked = (byte1 & 0x80) != 0; - int lengthField = byte1 & 0x7F; - - int headerSize = 2; - long payloadLength; - if (lengthField <= 125) { - payloadLength = lengthField; - } else if (lengthField == 126) { - while (pos < 4) { - int n = in.read(readBuf, pos, readBuf.length - pos); - if (n < 0) return; - pos += n; - } - payloadLength = ((readBuf[2] & 0xFF) << 8) | (readBuf[3] & 0xFF); - headerSize = 4; - } else { - while (pos < 10) { - int n = in.read(readBuf, pos, readBuf.length - pos); - if (n < 0) return; - pos += n; - } - payloadLength = 0; - for (int i = 0; i < 8; i++) { - payloadLength = (payloadLength << 8) | (readBuf[2 + i] & 0xFF); - } - headerSize = 10; - } - - if (masked) { - headerSize += 4; - } - - int totalFrameSize = (int) (headerSize + payloadLength); - - if (totalFrameSize > readBuf.length) { - byte[] newBuf = new byte[totalFrameSize]; - System.arraycopy(readBuf, 0, newBuf, 0, pos); - readBuf = newBuf; - } - - while (pos < totalFrameSize) { - int n = in.read(readBuf, pos, totalFrameSize - pos); - if (n < 0) return; - pos += n; - } - - if (opcode == WebSocketOpcode.CLOSE) { - return; - } - - if (opcode != WebSocketOpcode.BINARY && opcode != WebSocketOpcode.TEXT) { - continue; - } - - // Unmask payload in place - if (masked) { - int maskKeyOffset = headerSize - 4; - byte m0 = readBuf[maskKeyOffset]; - byte m1 = readBuf[maskKeyOffset + 1]; - byte m2 = readBuf[maskKeyOffset + 2]; - byte m3 = readBuf[maskKeyOffset + 3]; - for (int i = 0; i < (int) payloadLength; i++) { - switch (i & 3) { - case 0: - readBuf[headerSize + i] ^= m0; - break; - case 1: - readBuf[headerSize + i] ^= m1; - break; - case 2: - readBuf[headerSize + i] ^= m2; - break; - case 3: - readBuf[headerSize + i] ^= m3; - break; - } - } - } - - // Build complete unmasked response frame in a single array - byte[] responseHeader; - if (payloadLength <= 125) { - responseHeader = new byte[]{ - (byte) (0x80 | opcode), - (byte) payloadLength - }; - } else if (payloadLength <= 65535) { - responseHeader = new byte[]{ - (byte) (0x80 | opcode), - 126, - (byte) ((payloadLength >> 8) & 0xFF), - (byte) (payloadLength & 0xFF) - }; - } else { - responseHeader = new byte[10]; - responseHeader[0] = (byte) (0x80 | opcode); - responseHeader[1] = 127; - for (int i = 0; i < 8; i++) { - responseHeader[2 + i] = (byte) ((payloadLength >> (56 - i * 8)) & 0xFF); - } - } - - // Single write: header + payload together via BufferedOutputStream - out.write(responseHeader); - out.write(readBuf, headerSize, (int) payloadLength); - out.flush(); - } - } - - private void run() { - try (Socket client = serverSocket.accept()) { - client.setSoTimeout(10_000); - client.setTcpNoDelay(true); - InputStream in = client.getInputStream(); - OutputStream out = new BufferedOutputStream(client.getOutputStream()); - - completeHandshake(in, out); - echoFrames(in, out); - } catch (IOException e) { - if (!serverSocket.isClosed()) { - error.set(e); - } - } catch (Throwable t) { - error.set(t); - } - } - - void assertNoError() { - Throwable t = error.get(); - if (t != null) { - throw new AssertionError("echo server error", t); - } - } - - int getPort() { - return serverSocket.getLocalPort(); - } - - void start() { - thread = new Thread(this::run, "ws-echo-server"); - thread.setDaemon(true); - thread.start(); - } - } - - private static class ReceivedPayload implements WebSocketChannel.ResponseHandler { - int length; - long ptr; - - @Override - public void onBinaryMessage(long payload, int length) { - this.ptr = payload; - this.length = length; - } - - @Override - public void onClose(int code, String reason) { - } - } -} From f304d314643828c770a1fe09068c039b6f5251d6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:04:10 +0100 Subject: [PATCH 067/230] Detect decimal overflow on rescale in QwpTableBuffer addDecimal128 and addDecimal64 rescale values via a Decimal256 temporary, but only read the lower 128 or 64 bits back. If the rescaled value overflows into the upper bits, the data is silently truncated. Add fitsInStorageSizePow2() checks after rescaling and throw LineSenderException when the result no longer fits in the target storage size. Add tests for both Decimal128 and Decimal64 overflow paths. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 8 ++++ .../qwp/protocol/QwpTableBufferTest.java | 41 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index b4dc34a..4ba4f0d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -471,6 +471,10 @@ public void addDecimal128(Decimal128 value) { rescaleTemp.ofRaw(value.getHigh(), value.getLow()); rescaleTemp.setScale(value.getScale()); rescaleTemp.rescale(decimalScale); + if (!rescaleTemp.fitsInStorageSizePow2(4)) { + throw new LineSenderException("Decimal128 overflow: rescaling from scale " + + value.getScale() + " to " + decimalScale + " exceeds 128-bit capacity"); + } dataBuffer.putLong(rescaleTemp.getLh()); dataBuffer.putLong(rescaleTemp.getLl()); valueCount++; @@ -518,6 +522,10 @@ public void addDecimal64(Decimal64 value) { rescaleTemp.ofRaw(value.getValue()); rescaleTemp.setScale(value.getScale()); rescaleTemp.rescale(decimalScale); + if (!rescaleTemp.fitsInStorageSizePow2(3)) { + throw new LineSenderException("Decimal64 overflow: rescaling from scale " + + value.getScale() + " to " + decimalScale + " exceeds 64-bit capacity"); + } dataBuffer.putLong(rescaleTemp.getLl()); } else { dataBuffer.putLong(value.getValue()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 1c294d0..2ed4762 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -24,16 +24,57 @@ package io.questdb.client.test.cutlass.qwp.protocol; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal64; import org.junit.Test; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class QwpTableBufferTest { + @Test + public void testAddDecimal128RescaleOverflow() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL128, true); + // First row sets decimalScale = 10 + col.addDecimal128(Decimal128.fromLong(1, 10)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 10 + // multiplies by 10^10, which exceeds 128-bit capacity + try { + col.addDecimal128(new Decimal128(Long.MAX_VALUE / 2, Long.MAX_VALUE, 0)); + fail("Expected LineSenderException for 128-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal128 overflow: rescaling from scale 0 to 10 exceeds 128-bit capacity", e.getMessage()); + } + } + } + + @Test + public void testAddDecimal64RescaleOverflow() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + // First row sets decimalScale = 5 + col.addDecimal64(Decimal64.fromLong(1, 5)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 5 + // multiplies by 10^5 = 100_000, which exceeds 64-bit capacity + // Long.MAX_VALUE / 10 ≈ 9.2 * 10^17, * 10^5 ≈ 9.2 * 10^22 >> 2^63 + try { + col.addDecimal64(Decimal64.fromLong(Long.MAX_VALUE / 10, 0)); + fail("Expected LineSenderException for 64-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal64 overflow: rescaling from scale 0 to 5 exceeds 64-bit capacity", e.getMessage()); + } + } + } + @Test public void testAddDoubleArrayNullOnNonNullableColumn() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { From 1e452c699a604a28e942069c68c30b20a575b258 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:29:03 +0100 Subject: [PATCH 068/230] Port QwpGorillaEncoder tests from core Port ~30 Gorilla encoder tests from the core module to the java-questdb-client module, adapted for the client's off-heap memory API. The new test file includes helper methods for writing timestamps to off-heap memory and decoding delta-of-delta values for round-trip verification without porting the full decoder. Co-Authored-By: Claude Opus 4.6 --- .../qwp/protocol/QwpGorillaEncoderTest.java | 654 ++++++++++++++++++ 1 file changed, 654 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java new file mode 100644 index 0000000..0e2e553 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java @@ -0,0 +1,654 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpBitReader; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +public class QwpGorillaEncoderTest { + + @Test + public void testCalculateEncodedSizeConstantDelta() { + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + long src = putTimestamps(timestamps); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + // 8 (first) + 8 (second) + ceil(98 bits / 8) = 29 + int expectedBits = timestamps.length - 2; + int expectedSize = 8 + 8 + (expectedBits + 7) / 8; + Assert.assertEquals(expectedSize, size); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeEmpty() { + Assert.assertEquals(0, QwpGorillaEncoder.calculateEncodedSize(0, 0)); + } + + @Test + public void testCalculateEncodedSizeIdenticalDeltas() { + long[] ts = {100L, 200L, 300L}; // delta=100, DoD=0 + long src = putTimestamps(ts); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); + // 8 + 8 + 1 byte (1 bit padded to byte) = 17 + Assert.assertEquals(17, size); + } finally { + Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeOneTimestamp() { + long[] ts = {1000L}; + long src = putTimestamps(ts); + try { + Assert.assertEquals(8, QwpGorillaEncoder.calculateEncodedSize(src, 1)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeSmallDoD() { + long[] ts = {100L, 200L, 350L}; // delta0=100, delta1=150, DoD=50 + long src = putTimestamps(ts); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); + // 8 + 8 + 2 bytes (9 bits padded to bytes) = 18 + Assert.assertEquals(18, size); + } finally { + Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeTwoTimestamps() { + long[] ts = {1000L, 2000L}; + long src = putTimestamps(ts); + try { + Assert.assertEquals(16, QwpGorillaEncoder.calculateEncodedSize(src, 2)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaConstantDelta() { + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + long src = putTimestamps(timestamps); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaEmpty() { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(0, 0)); + } + + @Test + public void testCanUseGorillaLargeDoDOutOfRange() { + // DoD = 3_000_000_000 exceeds Integer.MAX_VALUE + long[] timestamps = { + 0L, + 1_000_000_000L, // delta=1_000_000_000 + 5_000_000_000L, // delta=4_000_000_000, DoD=3_000_000_000 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaNegativeLargeDoDOutOfRange() { + // DoD = -4_000_000_000 is less than Integer.MIN_VALUE + long[] timestamps = { + 10_000_000_000L, + 9_000_000_000L, // delta=-1_000_000_000 + 4_000_000_000L, // delta=-5_000_000_000, DoD=-4_000_000_000 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaOneTimestamp() { + long[] ts = {1000L}; + long src = putTimestamps(ts); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 1)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaTwoTimestamps() { + long[] ts = {1000L, 2000L}; + long src = putTimestamps(ts); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 2)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaVaryingDelta() { + long[] timestamps = { + 1_000_000_000L, + 1_000_001_000L, // delta=1000 + 1_000_002_100L, // DoD=100 + 1_000_003_500L, // DoD=300 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCompressionRatioConstantInterval() { + long[] timestamps = new long[1000]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = i * 1000L; + } + long src = putTimestamps(timestamps); + try { + int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + int uncompressedSize = timestamps.length * 8; + double ratio = (double) gorillaSize / uncompressedSize; + Assert.assertTrue("Compression ratio should be < 0.1 for constant interval", ratio < 0.1); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCompressionRatioRandomData() { + long[] timestamps = new long[100]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; + java.util.Random random = new java.util.Random(42); + for (int i = 2; i < timestamps.length; i++) { + timestamps[i] = timestamps[i - 1] + 1000 + random.nextInt(10_000) - 5000; + } + long src = putTimestamps(timestamps); + try { + int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + Assert.assertTrue("Size should be positive", gorillaSize > 0); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeDecodeBucketBoundaries() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + // t0=0, t1=10_000, delta0=10_000 + // For DoD=X: delta1 = 10_000+X, so t2 = 20_000+X + long[][] bucketTests = { + {0L, 10_000L, 20_000L}, // DoD = 0 (bucket 0) + {0L, 10_000L, 20_063L}, // DoD = 63 (bucket 1 max) + {0L, 10_000L, 19_936L}, // DoD = -64 (bucket 1 min) + {0L, 10_000L, 20_064L}, // DoD = 64 (bucket 2 start) + {0L, 10_000L, 19_935L}, // DoD = -65 (bucket 2 start) + {0L, 10_000L, 20_255L}, // DoD = 255 (bucket 2 max) + {0L, 10_000L, 19_744L}, // DoD = -256 (bucket 2 min) + {0L, 10_000L, 20_256L}, // DoD = 256 (bucket 3 start) + {0L, 10_000L, 19_743L}, // DoD = -257 (bucket 3 start) + {0L, 10_000L, 22_047L}, // DoD = 2047 (bucket 3 max) + {0L, 10_000L, 17_952L}, // DoD = -2048 (bucket 3 min) + {0L, 10_000L, 22_048L}, // DoD = 2048 (bucket 4 start) + {0L, 10_000L, 17_951L}, // DoD = -2049 (bucket 4 start) + {0L, 10_000L, 110_000L}, // DoD = 100_000 (bucket 4, large) + {0L, 10_000L, -80_000L}, // DoD = -100_000 (bucket 4, large) + }; + + for (long[] tc : bucketTests) { + long src = putTimestamps(tc); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, tc.length); + Assert.assertTrue("Failed to encode: " + java.util.Arrays.toString(tc), bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(tc[0], first); + Assert.assertEquals(tc[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long dod = decodeDoD(reader); + long delta = (second - first) + dod; + long decoded = second + delta; + Assert.assertEquals("Failed for: " + java.util.Arrays.toString(tc), tc[2], decoded); + } finally { + Unsafe.free(src, (long) tc.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + } + + @Test + public void testEncodeTimestampsEmpty() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, 0, 0); + Assert.assertEquals(0, bytesWritten); + } finally { + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsOneTimestamp() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long[] timestamps = {1_234_567_890L}; + long src = putTimestamps(timestamps); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 1); + Assert.assertEquals(8, bytesWritten); + Assert.assertEquals(1_234_567_890L, Unsafe.getUnsafe().getLong(dst)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripAllBuckets() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = new long[10]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; // delta = 1000 + timestamps[2] = 1_000_002_000L; // DoD=0 (bucket 0) + timestamps[3] = 1_000_003_050L; // DoD=50 (bucket 1) + timestamps[4] = 1_000_003_987L; // DoD=-113 (bucket 2) + timestamps[5] = 1_000_004_687L; // DoD=-237 (bucket 2) + timestamps[6] = 1_000_006_387L; // DoD=1000 (bucket 3) + timestamps[7] = 1_000_020_087L; // DoD=12000 (bucket 4) + timestamps[8] = 1_000_033_787L; // DoD=0 (bucket 0) + timestamps[9] = 1_000_047_487L; // DoD=0 (bucket 0) + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripConstantDelta() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripLargeDataset() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + int count = 10_000; + long[] timestamps = new long[count]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; + + java.util.Random random = new java.util.Random(42); + for (int i = 2; i < count; i++) { + long prevDelta = timestamps[i - 1] - timestamps[i - 2]; + int variation = (i % 10 == 0) ? random.nextInt(100) - 50 : 0; + timestamps[i] = timestamps[i - 1] + prevDelta + variation; + } + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, count) + 100; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, count); + Assert.assertTrue(bytesWritten > 0); + Assert.assertTrue("Should compress better than uncompressed", bytesWritten < count * 8); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < count; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) count * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripNegativeDoD() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = { + 1_000_000_000L, + 1_000_002_000L, // delta=2000 + 1_000_003_000L, // DoD=-1000 (bucket 3) + 1_000_003_500L, // DoD=-500 (bucket 2) + 1_000_003_600L, // DoD=-400 (bucket 2) + }; + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripVaryingDelta() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = { + 1_000_000_000L, + 1_000_001_000L, // delta=1000 + 1_000_002_000L, // DoD=0 (bucket 0) + 1_000_003_010L, // DoD=10 (bucket 1) + 1_000_004_120L, // DoD=100 (bucket 1) + 1_000_005_420L, // DoD=190 (bucket 2) + 1_000_007_720L, // DoD=1000 (bucket 3) + 1_000_020_020L, // DoD=10000 (bucket 4) + }; + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsTwoTimestamps() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long[] timestamps = {1_000_000_000L, 1_000_001_000L}; + long src = putTimestamps(timestamps); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 2); + Assert.assertEquals(16, bytesWritten); + Assert.assertEquals(1_000_000_000L, Unsafe.getUnsafe().getLong(dst)); + Assert.assertEquals(1_000_001_000L, Unsafe.getUnsafe().getLong(dst + 8)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testGetBitsRequired() { + // Bucket 0: 1 bit + Assert.assertEquals(1, QwpGorillaEncoder.getBitsRequired(0)); + + // Bucket 1: 9 bits (2 prefix + 7 value) + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(1)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-1)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(63)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-64)); + + // Bucket 2: 12 bits (3 prefix + 9 value) + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(64)); + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(255)); + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(-256)); + + // Bucket 3: 16 bits (4 prefix + 12 value) + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(256)); + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(2047)); + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(-2048)); + + // Bucket 4: 36 bits (4 prefix + 32 value) + Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(2048)); + Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(-2049)); + } + + @Test + public void testGetBucket12Bit() { + // DoD in [-2048, 2047] but outside [-256, 255] -> bucket 3 (16 bits) + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(256)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-257)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(2047)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2047)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2048)); + } + + @Test + public void testGetBucket32Bit() { + // DoD outside [-2048, 2047] -> bucket 4 (36 bits) + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(2048)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-2049)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(100_000)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-100_000)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MAX_VALUE)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MIN_VALUE)); + } + + @Test + public void testGetBucket7Bit() { + // DoD in [-64, 63] -> bucket 1 (9 bits) + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(1)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-1)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(63)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-63)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-64)); + } + + @Test + public void testGetBucket9Bit() { + // DoD in [-256, 255] but outside [-64, 63] -> bucket 2 (12 bits) + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(64)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-65)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(255)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-255)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-256)); + } + + @Test + public void testGetBucketZero() { + Assert.assertEquals(0, QwpGorillaEncoder.getBucket(0)); + } + + /** + * Decodes a delta-of-delta value from the bit stream, mirroring the + * core QwpGorillaDecoder.decodeDoD() logic. + */ + private static long decodeDoD(QwpBitReader reader) { + int bit = reader.readBit(); + if (bit == 0) { + return 0; + } + bit = reader.readBit(); + if (bit == 0) { + return reader.readSigned(7); + } + bit = reader.readBit(); + if (bit == 0) { + return reader.readSigned(9); + } + bit = reader.readBit(); + if (bit == 0) { + return reader.readSigned(12); + } + return reader.readSigned(32); + } + + /** + * Writes a Java array of timestamps to off-heap memory. + * + * @param timestamps the timestamps to write + * @return the address of the allocated memory (caller must free) + */ + private static long putTimestamps(long[] timestamps) { + long size = (long) timestamps.length * 8; + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + for (int i = 0; i < timestamps.length; i++) { + Unsafe.getUnsafe().putLong(address + (long) i * 8, timestamps[i]); + } + return address; + } +} From ec56390037128f7f5866f5568ff7d2959def003e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:30:38 +0100 Subject: [PATCH 069/230] Explain precondition in gorilla encoder --- .../client/cutlass/qwp/protocol/QwpGorillaEncoder.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index a21215a..0f57342 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -231,6 +231,11 @@ public void encodeDoD(long deltaOfDelta) { * - Remaining timestamps: bit-packed delta-of-delta * *

    + * Precondition: the caller must verify that {@link #canUseGorilla(long, int)} + * returns {@code true} before calling this method. The largest delta-of-delta + * bucket uses 32-bit signed encoding, so values outside the {@code int} range + * are silently truncated, producing corrupt output on decode. + *

    * Note: This method does NOT write the encoding flag byte. The caller is * responsible for writing the ENCODING_GORILLA flag before calling this method. * From c81c80f13c964a1ae5f52002f75fcab4c7b2d8a1 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:33:59 +0100 Subject: [PATCH 070/230] Add bounds assertion to jumpTo() Add a debug assertion in OffHeapAppendMemory.jumpTo() that validates the offset is non-negative and within the current append position. This is consistent with the assertion pattern used in MemoryPARWImpl.jumpTo() in core. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index 7388961..a30cf3c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -83,6 +83,7 @@ public long getAppendOffset() { * Used for truncateTo operations on column buffers. */ public void jumpTo(long offset) { + assert offset >= 0 && offset <= getAppendOffset(); appendAddress = pageAddress + offset; } From 99876bc5a6ee9b0acc2af1f75a5ea642b3e8cd7e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:40:57 +0100 Subject: [PATCH 071/230] Avoid BigDecimal allocation in decimalColumn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decimalColumn(CharSequence, CharSequence) previously allocated a BigDecimal and a new Decimal256 on every call. Replace this with a reusable Decimal256 field and its ofString() method, which parses the CharSequence directly into the mutable object in-place via DecimalParser — no intermediate objects needed. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 59e890a..1cc8e51 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -146,6 +146,7 @@ public class QwpWebSocketSender implements Sender { private boolean connected; // Track max global symbol ID used in current batch (for delta calculation) private int currentBatchMaxSymbolId = -1; + private final Decimal256 currentDecimal256 = new Decimal256(); private QwpTableBuffer currentTableBuffer; private String currentTableName; private long firstPendingRowTimeNanos; @@ -557,10 +558,9 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { checkNotClosed(); checkTableSelected(); try { - java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); - Decimal256 decimal = Decimal256.fromBigDecimal(bd); + currentDecimal256.ofString(value); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); - col.addDecimal256(decimal); + col.addDecimal256(currentDecimal256); } catch (Exception e) { throw new LineSenderException("Failed to parse decimal value: " + value, e); } From a999a909f8b6b0954170b6913661604ea3b1b5cc Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:41:08 +0100 Subject: [PATCH 072/230] Remove redundant local variable --- .../io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index e7efe6f..0117c0f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -181,8 +181,7 @@ public void ensureCapacity(int requiredCapacity) { } if (requiredCapacity > bufferCapacity) { int newCapacity = Math.max(bufferCapacity * 2, requiredCapacity); - long newPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); - bufferPtr = newPtr; + bufferPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); bufferCapacity = newCapacity; } } From 19ac1c6ff171708504090a1e0dbfc6026058b7a7 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 14:04:58 +0100 Subject: [PATCH 073/230] Fix UUID storage allocation in QwpTableBuffer allocateStorage() incorrectly grouped TYPE_UUID with the array types (TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY), causing it to allocate arrayDims and arrayCapture instead of a dataBuffer. Since addUuid() and addNull() for UUID both write to dataBuffer, any use of a UUID column would NPE. Move TYPE_UUID to the TYPE_DECIMAL128 case, which correctly allocates a 256-byte OffHeapAppendMemory for the 16-byte fixed-width data. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 4ba4f0d..7e21881 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -1194,6 +1194,7 @@ private void allocateStorage(byte type) { case TYPE_DOUBLE: dataBuffer = new OffHeapAppendMemory(128); break; + case TYPE_UUID: case TYPE_DECIMAL128: dataBuffer = new OffHeapAppendMemory(256); break; @@ -1212,7 +1213,6 @@ private void allocateStorage(byte type) { symbolDict = new CharSequenceIntHashMap(); symbolList = new ObjList<>(); break; - case TYPE_UUID: case TYPE_DOUBLE_ARRAY: case TYPE_LONG_ARRAY: arrayDims = new byte[16]; From c1500f8e2e37092b517ae38e478471a5730a837b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 14:23:33 +0100 Subject: [PATCH 074/230] Delete unused getSymbolsInRange() GlobalSymbolDictionary.getSymbolsInRange() had no production callers. Remove the method to reduce dead code. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/GlobalSymbolDictionary.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index ace342e..b8c1dfe 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -125,34 +125,6 @@ public String getSymbol(int id) { return idToSymbol.getQuick(id); } - /** - * Gets the symbols in the given ID range [fromId, toId). - *

    - * This is used to extract the delta for sending to the server. - * The range is inclusive of fromId and exclusive of toId. - * - * @param fromId start ID (inclusive) - * @param toId end ID (exclusive) - * @return array of symbols in the range, or empty array if range is invalid/empty - */ - public String[] getSymbolsInRange(int fromId, int toId) { - if (fromId < 0 || toId < fromId || fromId >= idToSymbol.size()) { - return new String[0]; - } - - int actualToId = Math.min(toId, idToSymbol.size()); - int count = actualToId - fromId; - if (count <= 0) { - return new String[0]; - } - - String[] result = new String[count]; - for (int i = 0; i < count; i++) { - result[i] = idToSymbol.getQuick(fromId + i); - } - return result; - } - /** * Checks if the dictionary is empty. * From 2a11d37c39500f08c34ab1d50a083f27ab857671 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 14:28:35 +0100 Subject: [PATCH 075/230] Move GlobalSymbolDictionaryTest to client The test class belongs with the code it tests. Move it from core's test tree into the client module under the matching package io.questdb.client.test.cutlass.qwp.client. Co-Authored-By: Claude Opus 4.6 --- .../client/GlobalSymbolDictionaryTest.java | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java new file mode 100644 index 0000000..c5eb84e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -0,0 +1,249 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class GlobalSymbolDictionaryTest { + + @Test + public void testAddSymbol_assignsSequentialIds() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertEquals(0, dict.getOrAddSymbol("AAPL")); + assertEquals(1, dict.getOrAddSymbol("GOOG")); + assertEquals(2, dict.getOrAddSymbol("MSFT")); + assertEquals(3, dict.getOrAddSymbol("TSLA")); + + assertEquals(4, dict.size()); + } + + @Test + public void testAddSymbol_deduplicatesSameSymbol() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + int id1 = dict.getOrAddSymbol("AAPL"); + int id2 = dict.getOrAddSymbol("AAPL"); + int id3 = dict.getOrAddSymbol("AAPL"); + + assertEquals(id1, id2); + assertEquals(id2, id3); + assertEquals(0, id1); + assertEquals(1, dict.size()); + } + + @Test + public void testClear() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + assertEquals(2, dict.size()); + + dict.clear(); + + assertTrue(dict.isEmpty()); + assertEquals(0, dict.size()); + assertFalse(dict.contains("AAPL")); + } + + @Test + public void testClear_thenAddRestartsFromZero() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.clear(); + + // New IDs should start from 0 + assertEquals(0, dict.getOrAddSymbol("MSFT")); + assertEquals(1, dict.getOrAddSymbol("TSLA")); + } + + @Test + public void testContains() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertFalse(dict.contains("AAPL")); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + + assertTrue(dict.contains("AAPL")); + assertTrue(dict.contains("GOOG")); + assertFalse(dict.contains("MSFT")); + assertFalse(dict.contains(null)); + } + + @Test + public void testCustomInitialCapacity() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(1024); + + // Should work normally + for (int i = 0; i < 100; i++) { + assertEquals(i, dict.getOrAddSymbol("SYM_" + i)); + } + assertEquals(100, dict.size()); + } + + @Test + public void testGetId_returnsCorrectId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.getOrAddSymbol("MSFT"); + + assertEquals(0, dict.getId("AAPL")); + assertEquals(1, dict.getId("GOOG")); + assertEquals(2, dict.getId("MSFT")); + } + + @Test + public void testGetId_returnsMinusOneForNull() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + assertEquals(-1, dict.getId(null)); + } + + @Test + public void testGetId_returnsMinusOneForUnknown() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + + assertEquals(-1, dict.getId("GOOG")); + assertEquals(-1, dict.getId("UNKNOWN")); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetOrAddSymbol_throwsForNull() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol(null); + } + + @Test + public void testGetSymbol_returnsCorrectSymbol() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.getOrAddSymbol("MSFT"); + + assertEquals("AAPL", dict.getSymbol(0)); + assertEquals("GOOG", dict.getSymbol(1)); + assertEquals("MSFT", dict.getSymbol(2)); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void testGetSymbol_throwsForInvalidId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + dict.getSymbol(1); // Only id 0 exists + } + + @Test(expected = IndexOutOfBoundsException.class) + public void testGetSymbol_throwsForNegativeId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + dict.getSymbol(-1); + } + + @Test + public void testIsEmpty() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertTrue(dict.isEmpty()); + + dict.getOrAddSymbol("AAPL"); + assertFalse(dict.isEmpty()); + } + + @Test + public void testLargeNumberOfSymbols() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Add 10000 symbols + for (int i = 0; i < 10000; i++) { + assertEquals(i, dict.getOrAddSymbol("SYMBOL_" + i)); + } + + assertEquals(10000, dict.size()); + + // Verify retrieval + for (int i = 0; i < 10000; i++) { + assertEquals("SYMBOL_" + i, dict.getSymbol(i)); + assertEquals(i, dict.getId("SYMBOL_" + i)); + } + } + + @Test + public void testMixedSymbolsAcrossTables() { + // Simulates symbols from multiple tables sharing the dictionary + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Table "trades": exchange column + int nyse = dict.getOrAddSymbol("NYSE"); // 0 + int nasdaq = dict.getOrAddSymbol("NASDAQ"); // 1 + + // Table "prices": currency column + int usd = dict.getOrAddSymbol("USD"); // 2 + int eur = dict.getOrAddSymbol("EUR"); // 3 + + // Table "orders": exchange column (reuses) + int nyse2 = dict.getOrAddSymbol("NYSE"); // Still 0 + + assertEquals(nyse, nyse2); + assertEquals(4, dict.size()); + + // All symbols accessible + assertEquals("NYSE", dict.getSymbol(nyse)); + assertEquals("NASDAQ", dict.getSymbol(nasdaq)); + assertEquals("USD", dict.getSymbol(usd)); + assertEquals("EUR", dict.getSymbol(eur)); + } + + @Test + public void testSpecialCharactersInSymbols() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol(""); // Empty string + dict.getOrAddSymbol(" "); // Space + dict.getOrAddSymbol("a b c"); // With spaces + dict.getOrAddSymbol("AAPL\u0000"); // With null char + dict.getOrAddSymbol("\u00E9"); // Unicode + dict.getOrAddSymbol("\uD83D\uDE00"); // Emoji + + assertEquals(6, dict.size()); + + assertEquals("", dict.getSymbol(0)); + assertEquals(" ", dict.getSymbol(1)); + assertEquals("a b c", dict.getSymbol(2)); + assertEquals("AAPL\u0000", dict.getSymbol(3)); + assertEquals("\u00E9", dict.getSymbol(4)); + assertEquals("\uD83D\uDE00", dict.getSymbol(5)); + } +} From 8630ead11661b6c5747cb8128f13b324317612eb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 15:05:16 +0100 Subject: [PATCH 076/230] Move DeltaSymbolDictionaryTest to client Move the 18 tests that exercise only client-side classes (GlobalSymbolDictionary, QwpWebSocketEncoder, QwpTableBuffer, QwpBufferWriter) from core into the client submodule. Drop three round-trip tests that depend on server-only classes (QwpStreamingDecoder, QwpMessageCursor, QwpSymbolColumnCursor). Adapt imports from io.questdb.std to io.questdb.client.std and from io.questdb.cutlass.qwp.protocol.QwpConstants to io.questdb.client.cutlass.qwp.protocol.QwpConstants. Remove the unused throws QwpParseException from encodeAndDecode and its helper. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/DeltaSymbolDictionaryTest.java | 598 ++++++++++++++++++ 1 file changed, 598 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java new file mode 100644 index 0000000..7001e84 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java @@ -0,0 +1,598 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Comprehensive tests for delta symbol dictionary encoding and decoding. + *

    + * Tests cover: + * - Multiple tables sharing the same global dictionary + * - Multiple batches with progressive symbol accumulation + * - Reconnection scenarios where the dictionary resets + * - Multiple symbol columns in the same table + * - Edge cases (empty batches, no symbols, etc.) + */ +public class DeltaSymbolDictionaryTest { + + @Test + public void testEdgeCase_batchWithNoSymbols() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table with only non-symbol columns + QwpTableBuffer batch = new QwpTableBuffer("metrics"); + QwpTableBuffer.ColumnBuffer valueCol = batch.getOrCreateColumn("value", TYPE_LONG, false); + valueCol.addLong(100L); + batch.nextRow(); + + // MaxId is -1 (no symbols) + int batchMaxId = -1; + + // Can still encode with delta dict (empty delta) + int size = encoder.encodeWithDeltaDict(batch, globalDict, -1, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify flag is set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + } + + @Test + public void testEdgeCase_duplicateSymbolsInBatch() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + QwpTableBuffer batch = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + // Same symbol used multiple times + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + Assert.assertEquals(3, batch.getRowCount()); + Assert.assertEquals(1, globalDict.size()); // Only 1 unique symbol + + int maxGlobalId = col.getMaxGlobalSymbolId(); + Assert.assertEquals(0, maxGlobalId); // Max ID is 0 (AAPL) + } + + @Test + public void testEdgeCase_emptyBatch() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Pre-populate dictionary and send + globalDict.getOrAddSymbol("AAPL"); + int maxSentSymbolId = 0; + + // Empty batch (no rows, no symbols used) + QwpTableBuffer emptyBatch = new QwpTableBuffer("test"); + Assert.assertEquals(0, emptyBatch.getRowCount()); + + // Delta should still work (deltaCount = 0) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 0; + Assert.assertEquals(1, deltaStart); + Assert.assertEquals(0, deltaCount); + } + + @Test + public void testEdgeCase_gapFill() { + // Client dictionary: AAPL(0), GOOG(1), MSFT(2), TSLA(3) + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); + globalDict.getOrAddSymbol("TSLA"); + + // Batch uses AAPL(0) and TSLA(3), skipping GOOG(1) and MSFT(2) + // Delta must include gap-fill: send all symbols from maxSentSymbolId+1 to batchMaxId + int maxSentSymbolId = -1; + int batchMaxId = 3; // TSLA + + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batchMaxId - maxSentSymbolId; + + // Must send symbols 0, 1, 2, 3 (even though 1, 2 aren't used in this batch) + Assert.assertEquals(0, deltaStart); + Assert.assertEquals(4, deltaCount); + + // This ensures server has contiguous dictionary + for (int id = deltaStart; id < deltaStart + deltaCount; id++) { + String symbol = globalDict.getSymbol(id); + Assert.assertNotNull("Symbol " + id + " should exist", symbol); + } + } + + @Test + public void testEdgeCase_largeSymbolDictionary() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Add 1000 unique symbols + for (int i = 0; i < 1000; i++) { + int id = globalDict.getOrAddSymbol("SYM_" + i); + Assert.assertEquals(i, id); + } + + Assert.assertEquals(1000, globalDict.size()); + + // Send first batch with symbols 0-99 + int maxSentSymbolId = 99; + + // Next batch uses symbols 0-199, delta is 100-199 + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 199 - maxSentSymbolId; + Assert.assertEquals(100, deltaStart); + Assert.assertEquals(100, deltaCount); + } + + @Test + public void testEdgeCase_nullSymbolValues() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + QwpTableBuffer batch = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, true); // nullable + + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + col.addSymbol(null); // NULL value + batch.nextRow(); + + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + Assert.assertEquals(3, batch.getRowCount()); + // Dictionary only has 1 symbol (AAPL), NULL doesn't add to dictionary + Assert.assertEquals(1, globalDict.size()); + } + + @Test + public void testEdgeCase_unicodeSymbols() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Unicode symbols + int id1 = globalDict.getOrAddSymbol("日本"); + int id2 = globalDict.getOrAddSymbol("中国"); + int id3 = globalDict.getOrAddSymbol("한국"); + int id4 = globalDict.getOrAddSymbol("Émoji🚀"); + + Assert.assertEquals(0, id1); + Assert.assertEquals(1, id2); + Assert.assertEquals(2, id3); + Assert.assertEquals(3, id4); + + Assert.assertEquals("日本", globalDict.getSymbol(0)); + Assert.assertEquals("中国", globalDict.getSymbol(1)); + Assert.assertEquals("한국", globalDict.getSymbol(2)); + Assert.assertEquals("Émoji🚀", globalDict.getSymbol(3)); + } + + @Test + public void testEdgeCase_veryLongSymbol() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Create a very long symbol (1000 chars) + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + sb.append('X'); + } + String longSymbol = sb.toString(); + + int id = globalDict.getOrAddSymbol(longSymbol); + Assert.assertEquals(0, id); + + String retrieved = globalDict.getSymbol(0); + Assert.assertEquals(longSymbol, retrieved); + Assert.assertEquals(1000, retrieved.length()); + } + + @Test + public void testMultipleBatches_encodeAndDecode() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + ObjList serverDict = new ObjList<>(); + int maxSentSymbolId = -1; + QwpTableBuffer batch1 = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col1 = batch1.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + int aaplId = clientDict.getOrAddSymbol("AAPL"); + int googId = clientDict.getOrAddSymbol("GOOG"); + col1.addSymbolWithGlobalId("AAPL", aaplId); + batch1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + batch1.nextRow(); + + int batch1MaxId = 1; + int size1 = encoder.encodeWithDeltaDict(batch1, clientDict, maxSentSymbolId, batch1MaxId, false); + Assert.assertTrue(size1 > 0); + maxSentSymbolId = batch1MaxId; + + // Decode on server side + QwpBufferWriter buf1 = encoder.getBuffer(); + decodeAndAccumulateDict(buf1.getBufferPtr(), size1, serverDict); + + // Verify server dictionary + Assert.assertEquals(2, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + QwpTableBuffer batch2 = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col2 = batch2.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + int msftId = clientDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Existing + batch2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); // New + batch2.nextRow(); + + int batch2MaxId = 2; + int size2 = encoder.encodeWithDeltaDict(batch2, clientDict, maxSentSymbolId, batch2MaxId, false); + Assert.assertTrue(size2 > 0); + maxSentSymbolId = batch2MaxId; + + // Decode batch 2 + QwpBufferWriter buf2 = encoder.getBuffer(); + decodeAndAccumulateDict(buf2.getBufferPtr(), size2, serverDict); + + // Server dictionary should now have 3 symbols + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(2)); + } + } + + @Test + public void testMultipleBatches_progressiveSymbolAccumulation() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Batch 1: AAPL, GOOG + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + int batch1MaxId = Math.max(aaplId, googId); + + // Simulate sending batch 1 - maxSentSymbolId = 1 after send + int maxSentSymbolId = batch1MaxId; // 1 + + // Batch 2: AAPL (existing), MSFT (new), TSLA (new) + globalDict.getOrAddSymbol("AAPL"); // Returns 0, already exists + int msftId = globalDict.getOrAddSymbol("MSFT"); + int tslaId = globalDict.getOrAddSymbol("TSLA"); + int batch2MaxId = Math.max(msftId, tslaId); + + // Delta for batch 2 should be [2, 3] (MSFT, TSLA) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batch2MaxId - maxSentSymbolId; + Assert.assertEquals(2, deltaStart); + Assert.assertEquals(2, deltaCount); + + // Simulate sending batch 2 + maxSentSymbolId = batch2MaxId; // 3 + + // Batch 3: All existing symbols (no delta needed) + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + int batch3MaxId = 1; // Max used is GOOG(1) + + deltaStart = maxSentSymbolId + 1; + deltaCount = Math.max(0, batch3MaxId - maxSentSymbolId); + Assert.assertEquals(4, deltaStart); + Assert.assertEquals(0, deltaCount); // No new symbols + } + + @Test + public void testMultipleTables_encodedInSameBatch() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Create two tables + QwpTableBuffer table1 = new QwpTableBuffer("trades"); + QwpTableBuffer table2 = new QwpTableBuffer("quotes"); + + // Table 1: ticker column + QwpTableBuffer.ColumnBuffer col1 = table1.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + col1.addSymbolWithGlobalId("AAPL", aaplId); + table1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + table1.nextRow(); + + // Table 2: symbol column (different name, but shares dictionary) + QwpTableBuffer.ColumnBuffer col2 = table2.getOrCreateColumn("symbol", TYPE_SYMBOL, false); + int msftId = globalDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); + table2.nextRow(); + + // Encode first table with delta dict + int confirmedMaxId = -1; + int batchMaxId = 2; // AAPL(0), GOOG(1), MSFT(2) + + int size = encoder.encodeWithDeltaDict(table1, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify delta section contains all 3 symbols + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // After header: deltaStart=0, deltaCount=3 + long pos = ptr + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + + @Test + public void testMultipleTables_multipleSymbolColumns() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + QwpTableBuffer table = new QwpTableBuffer("market_data"); + + // Column 1: exchange + QwpTableBuffer.ColumnBuffer exchangeCol = table.getOrCreateColumn("exchange", TYPE_SYMBOL, false); + int nyseId = globalDict.getOrAddSymbol("NYSE"); + int nasdaqId = globalDict.getOrAddSymbol("NASDAQ"); + + // Column 2: currency + QwpTableBuffer.ColumnBuffer currencyCol = table.getOrCreateColumn("currency", TYPE_SYMBOL, false); + int usdId = globalDict.getOrAddSymbol("USD"); + int eurId = globalDict.getOrAddSymbol("EUR"); + + // Column 3: ticker + QwpTableBuffer.ColumnBuffer tickerCol = table.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + + // Add row with all three columns + exchangeCol.addSymbolWithGlobalId("NYSE", nyseId); + currencyCol.addSymbolWithGlobalId("USD", usdId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); + table.nextRow(); + + exchangeCol.addSymbolWithGlobalId("NASDAQ", nasdaqId); + currencyCol.addSymbolWithGlobalId("EUR", eurId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table.nextRow(); + + // All symbols share the same global dictionary + Assert.assertEquals(5, globalDict.size()); + Assert.assertEquals("NYSE", globalDict.getSymbol(0)); + Assert.assertEquals("NASDAQ", globalDict.getSymbol(1)); + Assert.assertEquals("USD", globalDict.getSymbol(2)); + Assert.assertEquals("EUR", globalDict.getSymbol(3)); + Assert.assertEquals("AAPL", globalDict.getSymbol(4)); + } + + @Test + public void testMultipleTables_sharedGlobalDictionary() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table 1 uses symbols AAPL, GOOG + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + + // Table 2 uses symbols AAPL (reused), MSFT (new) + int aaplId2 = globalDict.getOrAddSymbol("AAPL"); // Should return same ID + int msftId = globalDict.getOrAddSymbol("MSFT"); + + // Verify deduplication + Assert.assertEquals(0, aaplId); + Assert.assertEquals(1, googId); + Assert.assertEquals(0, aaplId2); // Same as aaplId + Assert.assertEquals(2, msftId); + + // Total symbols should be 3 + Assert.assertEquals(3, globalDict.size()); + } + + @Test + public void testReconnection_fullDeltaAfterReconnect() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + + // First connection: add symbols + int aaplId = clientDict.getOrAddSymbol("AAPL"); + clientDict.getOrAddSymbol("GOOG"); + + // Send batch - maxSentSymbolId = 1 + int maxSentSymbolId = 1; + + // Reconnect - reset maxSentSymbolId + maxSentSymbolId = -1; + + // Create new batch using existing symbols + QwpTableBuffer batch = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + // Encode - should send full delta (all symbols from 0) + int size = encoder.encodeWithDeltaDict(batch, clientDict, maxSentSymbolId, 1, false); + Assert.assertTrue(size > 0); + + // Verify deltaStart is 0 + QwpBufferWriter buf = encoder.getBuffer(); + long pos = buf.getBufferPtr() + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + + @Test + public void testReconnection_resetsWatermark() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Build up dictionary and "send" some symbols + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); + + int maxSentSymbolId = 2; + + // Simulate reconnection - reset maxSentSymbolId + maxSentSymbolId = -1; + Assert.assertEquals(-1, maxSentSymbolId); + + // Global dictionary is NOT cleared (it's client-side) + Assert.assertEquals(3, globalDict.size()); + + // Next batch must send full delta from 0 + int deltaStart = maxSentSymbolId + 1; + Assert.assertEquals(0, deltaStart); + } + + @Test + public void testReconnection_serverDictionaryCleared() { + ObjList serverDict = new ObjList<>(); + + // Simulate first connection + serverDict.add("AAPL"); + serverDict.add("GOOG"); + Assert.assertEquals(2, serverDict.size()); + + // Simulate reconnection - server clears dictionary + serverDict.clear(); + Assert.assertEquals(0, serverDict.size()); + + // New connection starts fresh + serverDict.add("MSFT"); + Assert.assertEquals(1, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(0)); + } + + @Test + public void testServerSide_accumulateDelta() { + ObjList serverDict = new ObjList<>(); + + // First batch: symbols 0-2 + accumulateDelta(serverDict, 0, new String[]{"AAPL", "GOOG", "MSFT"}); + + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); + + // Second batch: symbols 3-4 + accumulateDelta(serverDict, 3, new String[]{"TSLA", "AMZN"}); + + Assert.assertEquals(5, serverDict.size()); + Assert.assertEquals("TSLA", serverDict.get(3)); + Assert.assertEquals("AMZN", serverDict.get(4)); + + // Third batch: no new symbols (empty delta) + accumulateDelta(serverDict, 5, new String[]{}); + Assert.assertEquals(5, serverDict.size()); + } + + @Test + public void testServerSide_resolveSymbol() { + ObjList serverDict = new ObjList<>(); + serverDict.add("AAPL"); + serverDict.add("GOOG"); + serverDict.add("MSFT"); + + // Resolve by global ID + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); + } + + private void accumulateDelta(ObjList serverDict, int deltaStart, String[] symbols) { + // Ensure capacity + while (serverDict.size() < deltaStart + symbols.length) { + serverDict.add(null); + } + // Add symbols + for (int i = 0; i < symbols.length; i++) { + serverDict.setQuick(deltaStart + i, symbols[i]); + } + } + + private void decodeAndAccumulateDict(long ptr, int size, ObjList serverDict) { + // Parse header + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + if ((flags & FLAG_DELTA_SYMBOL_DICT) == 0) { + return; // No delta dict + } + + // Parse delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart + int deltaStart = readVarint(pos); + pos += 1; // Assuming single-byte varint + + // Read deltaCount + int deltaCount = readVarint(pos); + pos += 1; + + // Ensure capacity + while (serverDict.size() < deltaStart + deltaCount) { + serverDict.add(null); + } + + // Read symbols + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(pos); + pos += 1; + + byte[] bytes = new byte[len]; + for (int j = 0; j < len; j++) { + bytes[j] = Unsafe.getUnsafe().getByte(pos + j); + } + pos += len; + + serverDict.setQuick(deltaStart + i, new String(bytes, java.nio.charset.StandardCharsets.UTF_8)); + } + } + + private int readVarint(long address) { + byte b = Unsafe.getUnsafe().getByte(address); + if ((b & 0x80) == 0) { + return b & 0x7F; + } + // For simplicity, only handle single-byte varints in tests + return b & 0x7F; + } +} From 793f6f627d0da211ea51f4fd1e1bb39f1dcc09fa Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 15:27:03 +0100 Subject: [PATCH 077/230] Move 5 pure client tests to client submodule Move NativeBufferWriterTest, MicrobatchBufferTest, QwpWebSocketEncoderTest, QwpWebSocketSenderTest, and WebSocketSendQueueTest from core's websocket test directory into the java-questdb-client submodule. These tests only exercise classes in io.questdb.client.* and do not depend on core-module server-side classes. For NativeBufferWriterTest and MicrobatchBufferTest, the client module already had versions with different test methods, so the core methods are merged into the existing files. For the other three, new files are created with package and import paths adjusted to the client module (io.questdb.std -> io.questdb.client.std, etc.). Four integration tests that span both client and core modules remain in the core test tree. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/MicrobatchBufferTest.java | 639 +++++++++ .../qwp/client/NativeBufferWriterTest.java | 335 ++++- .../qwp/client/QwpWebSocketEncoderTest.java | 1254 +++++++++++++++++ .../qwp/client/QwpWebSocketSenderTest.java | 458 ++++++ .../qwp/client/WebSocketSendQueueTest.java | 339 +++++ 5 files changed, 2991 insertions(+), 34 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java index a1f1ecf..b5f4355 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -25,18 +25,98 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; import org.junit.Test; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class MicrobatchBufferTest { + @Test + public void testAwaitRecycled() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + AtomicBoolean recycled = new AtomicBoolean(false); + CountDownLatch started = new CountDownLatch(1); + + Thread waiter = new Thread(() -> { + started.countDown(); + buffer.awaitRecycled(); + recycled.set(true); + }); + waiter.start(); + + started.await(); + Thread.sleep(50); // Give waiter time to start waiting + Assert.assertFalse(recycled.get()); + + buffer.markRecycled(); + waiter.join(1000); + + Assert.assertTrue(recycled.get()); + } + }); + } + + @Test + public void testAwaitRecycledWithTimeout() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + // Should timeout + boolean result = buffer.awaitRecycled(50, TimeUnit.MILLISECONDS); + Assert.assertFalse(result); + + buffer.markRecycled(); + + // Should succeed immediately now + result = buffer.awaitRecycled(50, TimeUnit.MILLISECONDS); + Assert.assertTrue(result); + } + }); + } + + @Test + public void testBatchIdIncrementsOnReset() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + long id1 = buffer.getBatchId(); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + long id2 = buffer.getBatchId(); + Assert.assertNotEquals(id1, id2); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + long id3 = buffer.getBatchId(); + Assert.assertNotEquals(id2, id3); + } + }); + } + @Test public void testConcurrentBatchIdUniqueness() throws Exception { int threadCount = 8; @@ -122,4 +202,563 @@ public void testConcurrentResetBatchIdUniqueness() throws Exception { batchIds.size() ); } + + @Test + public void testConcurrentStateTransitions() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + AtomicReference error = new AtomicReference<>(); + CountDownLatch userDone = new CountDownLatch(1); + CountDownLatch ioDone = new CountDownLatch(1); + + // Simulate user thread + Thread userThread = new Thread(() -> { + try { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + buffer.seal(); + userDone.countDown(); + + // Wait for I/O thread to recycle + buffer.awaitRecycled(); + + // Reset and write again + buffer.reset(); + buffer.writeByte((byte) 2); + } catch (Throwable t) { + error.set(t); + } + }); + + // Simulate I/O thread + Thread ioThread = new Thread(() -> { + try { + userDone.await(); + buffer.markSending(); + + // Simulate sending + Thread.sleep(10); + + buffer.markRecycled(); + ioDone.countDown(); + } catch (Throwable t) { + error.set(t); + } + }); + + userThread.start(); + ioThread.start(); + + userThread.join(1000); + ioThread.join(1000); + + Assert.assertNull(error.get()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertEquals(1, buffer.getBufferPos()); + } + }); + } + + @Test + public void testConstructionWithCustomThresholds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 100, 4096, 1_000_000_000L)) { + Assert.assertEquals(1024, buffer.getBufferCapacity()); + Assert.assertTrue(buffer.isFilling()); + } + }); + } + + @Test + public void testConstructionWithDefaultThresholds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(1024, buffer.getBufferCapacity()); + Assert.assertEquals(0, buffer.getBufferPos()); + Assert.assertEquals(0, buffer.getRowCount()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.hasData()); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructionWithNegativeCapacity() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer ignored = new MicrobatchBuffer(-1)) { + Assert.fail("Should have thrown"); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructionWithZeroCapacity() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer ignored = new MicrobatchBuffer(0)) { + Assert.fail("Should have thrown"); + } + }); + } + + @Test + public void testEnsureCapacityGrows() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.ensureCapacity(2000); + Assert.assertTrue(buffer.getBufferCapacity() >= 2000); + } + }); + } + + @Test + public void testEnsureCapacityNoGrowth() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.ensureCapacity(512); + Assert.assertEquals(1024, buffer.getBufferCapacity()); // No change + } + }); + } + + @Test + public void testFirstRowTimeIsRecorded() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(0, buffer.getAgeNanos()); + + buffer.incrementRowCount(); + long age1 = buffer.getAgeNanos(); + Assert.assertTrue(age1 >= 0); + + Thread.sleep(10); + + long age2 = buffer.getAgeNanos(); + Assert.assertTrue(age2 > age1); + } + }); + } + + @Test + public void testFullStateLifecycle() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + // FILLING + Assert.assertTrue(buffer.isFilling()); + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + // FILLING -> SEALED + buffer.seal(); + Assert.assertTrue(buffer.isSealed()); + + // SEALED -> SENDING + buffer.markSending(); + Assert.assertTrue(buffer.isSending()); + + // SENDING -> RECYCLED + buffer.markRecycled(); + Assert.assertTrue(buffer.isRecycled()); + + // RECYCLED -> FILLING (reset) + buffer.reset(); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.hasData()); + } + }); + } + + @Test + public void testIncrementRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(0, buffer.getRowCount()); + buffer.incrementRowCount(); + Assert.assertEquals(1, buffer.getRowCount()); + buffer.incrementRowCount(); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testIncrementRowCountWhenSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.incrementRowCount(); // Should throw + } + }); + } + + @Test + public void testInitialState() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(MicrobatchBuffer.STATE_FILLING, buffer.getState()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.isSealed()); + Assert.assertFalse(buffer.isSending()); + Assert.assertFalse(buffer.isRecycled()); + Assert.assertFalse(buffer.isInUse()); + } + }); + } + + @Test + public void testMarkRecycledTransition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + + Assert.assertEquals(MicrobatchBuffer.STATE_RECYCLED, buffer.getState()); + Assert.assertTrue(buffer.isRecycled()); + Assert.assertFalse(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testMarkRecycledWhenNotSending() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markRecycled(); // Should throw - not sending + } + }); + } + + @Test + public void testMarkSendingTransition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + Assert.assertEquals(MicrobatchBuffer.STATE_SENDING, buffer.getState()); + Assert.assertTrue(buffer.isSending()); + Assert.assertTrue(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testMarkSendingWhenNotSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.markSending(); // Should throw - not sealed + } + }); + } + + @Test + public void testResetFromRecycled() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + long oldBatchId = buffer.getBatchId(); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + Assert.assertTrue(buffer.isFilling()); + Assert.assertEquals(0, buffer.getBufferPos()); + Assert.assertEquals(0, buffer.getRowCount()); + Assert.assertNotEquals(oldBatchId, buffer.getBatchId()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testResetWhenSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.reset(); // Should throw + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testResetWhenSending() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + buffer.reset(); // Should throw + } + }); + } + + @Test + public void testRollbackSealForRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + buffer.seal(); + Assert.assertTrue(buffer.isSealed()); + + buffer.rollbackSealForRetry(); + Assert.assertTrue(buffer.isFilling()); + + // Verify the same batch remains writable after rollback. + buffer.writeByte((byte) 2); + buffer.incrementRowCount(); + Assert.assertEquals(2, buffer.getBufferPos()); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testRollbackSealWhenNotSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.rollbackSealForRetry(); // Should throw - not sealed + } + }); + } + + @Test + public void testSealTransition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.seal(); + + Assert.assertEquals(MicrobatchBuffer.STATE_SEALED, buffer.getState()); + Assert.assertFalse(buffer.isFilling()); + Assert.assertTrue(buffer.isSealed()); + Assert.assertTrue(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testSealWhenNotFilling() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.seal(); // Should throw + } + }); + } + + @Test + public void testSetBufferPos() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(100); + Assert.assertEquals(100, buffer.getBufferPos()); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetBufferPosNegative() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(-1); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetBufferPosOutOfBounds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(2000); + } + }); + } + + @Test + public void testShouldFlushAgeLimit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // 50ms timeout + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 0, 50_000_000L)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); + + Thread.sleep(60); + + Assert.assertTrue(buffer.shouldFlush()); + Assert.assertTrue(buffer.isAgeLimitExceeded()); + } + }); + } + + @Test + public void testShouldFlushByteLimit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 10, 0)) { + for (int i = 0; i < 9; i++) { + buffer.writeByte((byte) i); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); + } + buffer.writeByte((byte) 9); + buffer.incrementRowCount(); + Assert.assertTrue(buffer.shouldFlush()); + Assert.assertTrue(buffer.isByteLimitExceeded()); + } + }); + } + + @Test + public void testShouldFlushEmptyBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 1, 1, 1)) { + Assert.assertFalse(buffer.shouldFlush()); // Empty buffer never flushes + } + }); + } + + @Test + public void testShouldFlushRowLimit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 5, 0, 0)) { + for (int i = 0; i < 4; i++) { + buffer.writeByte((byte) i); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); + } + buffer.writeByte((byte) 4); + buffer.incrementRowCount(); + Assert.assertTrue(buffer.shouldFlush()); + Assert.assertTrue(buffer.isRowLimitExceeded()); + } + }); + } + + @Test + public void testShouldFlushWithNoThresholds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); // No thresholds set + } + }); + } + + @Test + public void testStateName() { + Assert.assertEquals("FILLING", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_FILLING)); + Assert.assertEquals("SEALED", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_SEALED)); + Assert.assertEquals("SENDING", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_SENDING)); + Assert.assertEquals("RECYCLED", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_RECYCLED)); + Assert.assertEquals("UNKNOWN(99)", MicrobatchBuffer.stateName(99)); + } + + @Test + public void testToString() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + String str = buffer.toString(); + Assert.assertTrue(str.contains("MicrobatchBuffer")); + Assert.assertTrue(str.contains("state=FILLING")); + Assert.assertTrue(str.contains("rows=1")); + Assert.assertTrue(str.contains("bytes=1")); + } + }); + } + + @Test + public void testWriteBeyondInitialCapacity() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(16)) { + // Write more than initial capacity + for (int i = 0; i < 100; i++) { + buffer.writeByte((byte) i); + } + Assert.assertEquals(100, buffer.getBufferPos()); + Assert.assertTrue(buffer.getBufferCapacity() >= 100); + + // Verify data integrity after growth + for (int i = 0; i < 100; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) i, read); + } + } + }); + } + + @Test + public void testWriteByte() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 0x42); + Assert.assertEquals(1, buffer.getBufferPos()); + Assert.assertTrue(buffer.hasData()); + + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr()); + Assert.assertEquals((byte) 0x42, read); + } + }); + } + + @Test + public void testWriteFromNativeMemory() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long src = Unsafe.malloc(10, MemoryTag.NATIVE_DEFAULT); + try { + // Fill source with test data + for (int i = 0; i < 10; i++) { + Unsafe.getUnsafe().putByte(src + i, (byte) (i + 100)); + } + + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.write(src, 10); + Assert.assertEquals(10, buffer.getBufferPos()); + + // Verify data + for (int i = 0; i < 10; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) (i + 100), read); + } + } + } finally { + Unsafe.free(src, 10, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testWriteMultipleBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + for (int i = 0; i < 100; i++) { + buffer.writeByte((byte) i); + } + Assert.assertEquals(100, buffer.getBufferPos()); + + // Verify data + for (int i = 0; i < 100; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) i, read); + } + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testWriteWhenSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.writeByte((byte) 1); // Should throw + } + }); + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index bfa2bc7..c77f8d3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -30,9 +30,7 @@ import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; public class NativeBufferWriterTest { @@ -45,6 +43,66 @@ public void testEnsureCapacityGrowsBuffer() { } } + @Test + public void testGrowBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // Write more than initial capacity + for (int i = 0; i < 100; i++) { + writer.putLong(i); + } + Assert.assertEquals(800, writer.getPosition()); + // Verify data + for (int i = 0; i < 100; i++) { + Assert.assertEquals(i, Unsafe.getUnsafe().getLong(writer.getBufferPtr() + i * 8)); + } + } + } + + @Test + public void testMultipleWrites() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 'I'); + writer.putByte((byte) 'L'); + writer.putByte((byte) 'P'); + writer.putByte((byte) '4'); + writer.putByte((byte) 1); // Version + writer.putByte((byte) 0); // Flags + writer.putShort((short) 1); // Table count + writer.putInt(0); // Payload length placeholder + + Assert.assertEquals(12, writer.getPosition()); + + // Verify ILP4 header + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + } + } + + @Test + public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + } + + @Test + public void testPatchInt() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0); // Placeholder at offset 0 + writer.putInt(100); // At offset 4 + writer.patchInt(0, 42); // Patch first int + Assert.assertEquals(42, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + Assert.assertEquals(100, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + } + @Test public void testPatchIntAtLastValidOffset() { try (NativeBufferWriter writer = new NativeBufferWriter(16)) { @@ -68,26 +126,22 @@ public void testPatchIntAtValidOffset() { } @Test - public void testSkipAdvancesPosition() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - writer.skip(4); - assertEquals(4, writer.getPosition()); - writer.skip(8); - assertEquals(12, writer.getPosition()); - } - } + public void testPutBlockOfBytes() { + try (NativeBufferWriter writer = new NativeBufferWriter(); + NativeBufferWriter source = new NativeBufferWriter()) { + // Prepare source data + source.putByte((byte) 1); + source.putByte((byte) 2); + source.putByte((byte) 3); + source.putByte((byte) 4); - @Test - public void testSkipBeyondCapacityGrowsBuffer() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - // skip past the 16-byte buffer — must grow, not corrupt memory - writer.skip(32); - assertEquals(32, writer.getPosition()); - assertTrue(writer.getCapacity() >= 32); - // writing after the skip must also succeed - writer.putInt(0xCAFE); - assertEquals(36, writer.getPosition()); - assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + // Copy to writer + writer.putBlockOfBytes(source.getBufferPtr(), 4); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals((byte) 1, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 2, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 3, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 4, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); } } @@ -149,18 +203,6 @@ public void testPutUtf8LoneSurrogateMatchesUtf8Length() { } } - @Test - public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { - // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 - assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); - // Lone high surrogate at end: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); - // Lone low surrogate: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); - // Valid pair still works: 4 bytes - assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); - } - @Test public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 @@ -173,6 +215,43 @@ public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { assertEquals(4, QwpBufferWriter.utf8Length("\uD83D\uDE00")); } + @Test + public void testReset() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(12345); + Assert.assertEquals(4, writer.getPosition()); + writer.reset(); + Assert.assertEquals(0, writer.getPosition()); + // Can write again + writer.putByte((byte) 0xFF); + Assert.assertEquals(1, writer.getPosition()); + } + } + + @Test + public void testSkipAdvancesPosition() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.skip(4); + assertEquals(4, writer.getPosition()); + writer.skip(8); + assertEquals(12, writer.getPosition()); + } + } + + @Test + public void testSkipBeyondCapacityGrowsBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // skip past the 16-byte buffer — must grow, not corrupt memory + writer.skip(32); + assertEquals(32, writer.getPosition()); + assertTrue(writer.getCapacity() >= 32); + // writing after the skip must also succeed + writer.putInt(0xCAFE); + assertEquals(36, writer.getPosition()); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + } + } + @Test public void testSkipThenPatchInt() { try (NativeBufferWriter writer = new NativeBufferWriter(8)) { @@ -185,4 +264,192 @@ public void testSkipThenPatchInt() { assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); } } + + @Test + public void testUtf8Length() { + Assert.assertEquals(0, NativeBufferWriter.utf8Length(null)); + Assert.assertEquals(0, NativeBufferWriter.utf8Length("")); + Assert.assertEquals(5, NativeBufferWriter.utf8Length("hello")); + Assert.assertEquals(2, NativeBufferWriter.utf8Length("ñ")); + Assert.assertEquals(3, NativeBufferWriter.utf8Length("€")); + } + + @Test + public void testWriteByte() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 0x42); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0x42, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testWriteDouble() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putDouble(3.14159265359); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(3.14159265359, Unsafe.getUnsafe().getDouble(writer.getBufferPtr()), 0.0000000001); + } + } + + @Test + public void testWriteEmptyString() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(""); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testWriteFloat() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putFloat(3.14f); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals(3.14f, Unsafe.getUnsafe().getFloat(writer.getBufferPtr()), 0.0001f); + } + } + + @Test + public void testWriteInt() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0x12345678); + Assert.assertEquals(4, writer.getPosition()); + // Little-endian + Assert.assertEquals(0x12345678, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + } + } + + @Test + public void testWriteLong() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLong(0x123456789ABCDEF0L); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(0x123456789ABCDEF0L, Unsafe.getUnsafe().getLong(writer.getBufferPtr())); + } + } + + @Test + public void testWriteLongBigEndian() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLongBE(0x0102030405060708L); + Assert.assertEquals(8, writer.getPosition()); + // Check big-endian byte order + long ptr = writer.getBufferPtr(); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 0x02, Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 0x03, Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) 0x04, Unsafe.getUnsafe().getByte(ptr + 3)); + Assert.assertEquals((byte) 0x05, Unsafe.getUnsafe().getByte(ptr + 4)); + Assert.assertEquals((byte) 0x06, Unsafe.getUnsafe().getByte(ptr + 5)); + Assert.assertEquals((byte) 0x07, Unsafe.getUnsafe().getByte(ptr + 6)); + Assert.assertEquals((byte) 0x08, Unsafe.getUnsafe().getByte(ptr + 7)); + } + } + + @Test + public void testWriteNullString() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(null); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testWriteShort() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putShort((short) 0x1234); + Assert.assertEquals(2, writer.getPosition()); + // Little-endian + Assert.assertEquals((byte) 0x34, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x12, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testWriteString() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString("hello"); + // Length (1 byte varint) + 5 bytes + Assert.assertEquals(6, writer.getPosition()); + // Check length + Assert.assertEquals((byte) 5, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + // Check content + Assert.assertEquals((byte) 'h', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'e', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 4)); + Assert.assertEquals((byte) 'o', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 5)); + } + } + + @Test + public void testWriteUtf8Ascii() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putUtf8("ABC"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 'A', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'B', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'C', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + } + + @Test + public void testWriteUtf8ThreeByte() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // € is 3 bytes in UTF-8 + writer.putUtf8("€"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 0xE2, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x82, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0xAC, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + } + + @Test + public void testWriteUtf8TwoByte() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // ñ is 2 bytes in UTF-8 + writer.putUtf8("ñ"); + Assert.assertEquals(2, writer.getPosition()); + Assert.assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0xB1, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testWriteVarintLarge() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Test larger value + writer.putVarint(16384); + Assert.assertEquals(3, writer.getPosition()); + // LEB128: 16384 = 0x80 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + } + + @Test + public void testWriteVarintMedium() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Two bytes for 128 + writer.putVarint(128); + Assert.assertEquals(2, writer.getPosition()); + // LEB128: 128 = 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testWriteVarintSmall() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Single byte for values < 128 + writer.putVarint(127); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 127, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java new file mode 100644 index 0000000..5a298e9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -0,0 +1,1254 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Unit tests for QwpWebSocketEncoder. + */ +public class QwpWebSocketEncoderTest { + + @Test + public void testBufferResetAndReuse() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // First batch + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + int size1 = encoder.encode(buffer, false); + + // Reset and second batch + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i * 2); + buffer.nextRow(); + } + int size2 = encoder.encode(buffer, false); + + Assert.assertTrue(size1 > size2); // More rows = larger + Assert.assertEquals(50, buffer.getRowCount()); + } + } + + @Test + public void testEncode2DDoubleArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncode2DLongArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[][]{{1L, 2L}, {3L, 4L}}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncode3DDoubleArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("tensor", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][][]{ + {{1.0, 2.0}, {3.0, 4.0}}, + {{5.0, 6.0}, {7.0, 8.0}} + }); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeAllBasicTypesInOneRow() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("all_types"); + + buffer.getOrCreateColumn("b", TYPE_BOOLEAN, false).addBoolean(true); + buffer.getOrCreateColumn("by", TYPE_BYTE, false).addByte((byte) 42); + buffer.getOrCreateColumn("sh", TYPE_SHORT, false).addShort((short) 1000); + buffer.getOrCreateColumn("i", TYPE_INT, false).addInt(100000); + buffer.getOrCreateColumn("l", TYPE_LONG, false).addLong(1000000000L); + buffer.getOrCreateColumn("f", TYPE_FLOAT, false).addFloat(3.14f); + buffer.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(3.14159265); + buffer.getOrCreateColumn("s", TYPE_STRING, true).addString("test"); + buffer.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbol("AAPL"); + buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(1, buffer.getRowCount()); + } + } + + @Test + public void testEncodeAllBooleanValues() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("flag", TYPE_BOOLEAN, false); + for (int i = 0; i < 100; i++) { + col.addBoolean(i % 2 == 0); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + } + + @Test + public void testEncodeDecimal128() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("amount", TYPE_DECIMAL128, false); + col.addDecimal128(io.questdb.client.std.Decimal128.fromLong(123456789012345L, 4)); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeDecimal256() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("bignum", TYPE_DECIMAL256, false); + col.addDecimal256(io.questdb.client.std.Decimal256.fromLong(Long.MAX_VALUE, 6)); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeDecimal64() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeDoubleArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[]{1.0, 2.0, 3.0}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeEmptyString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(""); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeEmptyTableName() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + // Edge case: empty table name (probably invalid but let's verify encoding works) + QwpTableBuffer buffer = new QwpTableBuffer(""); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 0); + } + } + + @Test + public void testEncodeLargeArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Large 1D array + double[] largeArray = new double[1000]; + for (int i = 0; i < 1000; i++) { + largeArray[i] = i * 1.5; + } + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(largeArray); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 8000); // At least 8 bytes per double + } + } + + @Test + public void testEncodeLargeRowCount() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("metrics"); + + for (int i = 0; i < 10000; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10000, buffer.getRowCount()); + } + } + + @Test + public void testEncodeLongArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[]{1L, 2L, 3L}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== SYMBOL COLUMN TESTS ==================== + + @Test + public void testEncodeLongString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + String sb = "a".repeat(10000); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("data", TYPE_STRING, true); + col.addString(sb); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 10000); + } + } + + @Test + public void testEncodeMaxMinLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(Long.MAX_VALUE); + buffer.nextRow(); + + col.addLong(Long.MIN_VALUE); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(2, buffer.getRowCount()); + } + } + + @Test + public void testEncodeMixedColumnTypes() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("events"); + + // Add columns of different types + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server1"); + + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(42); + + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(3.14); + + QwpTableBuffer.ColumnBuffer boolCol = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + boolCol.addBoolean(true); + + QwpTableBuffer.ColumnBuffer stringCol = buffer.getOrCreateColumn("message", TYPE_STRING, true); + stringCol.addString("hello world"); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeMixedColumnsMultipleRows() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("events"); + + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server" + (i % 5)); + + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(i * 10); + + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(i * 1.5); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L + i); + + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(50, buffer.getRowCount()); + } + } + + // ==================== UUID COLUMN TESTS ==================== + + @Test + public void testEncodeMultipleColumns() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("weather"); + + // Add multiple columns + QwpTableBuffer.ColumnBuffer tempCol = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + tempCol.addDouble(23.5); + + QwpTableBuffer.ColumnBuffer humCol = buffer.getOrCreateColumn("humidity", TYPE_LONG, false); + humCol.addLong(65); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + } + } + + @Test + public void testEncodeMultipleDecimal64() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); + + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(67890L, 2)); // 678.90 + buffer.nextRow(); + + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(11111L, 2)); // 111.11 + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + } + + // ==================== DECIMAL COLUMN TESTS ==================== + + @Test + public void testEncodeMultipleRows() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("metrics"); + + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer valCol = buffer.getOrCreateColumn("value", TYPE_LONG, false); + valCol.addLong(i); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L + i); + + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + } + + @Test + public void testEncodeMultipleSymbolsSameDictionary() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); + + col.addSymbol("server1"); // Same symbol + buffer.nextRow(); + + col.addSymbol("server2"); // Different symbol + buffer.nextRow(); + + col.addSymbol("server1"); // Back to first + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + } + + @Test + public void testEncodeMultipleUuids() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + for (int i = 0; i < 10; i++) { + col.addUuid(i * 1000L, i * 2000L); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10, buffer.getRowCount()); + } + } + + @Test + public void testEncodeNaNDouble() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.NaN); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== ARRAY COLUMN TESTS ==================== + + @Test + public void testEncodeNegativeLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(-123456789L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeNullableColumnWithNull() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Nullable column with null + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(null); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeNullableColumnWithValue() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Nullable column with a value + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeNullableSymbolWithNull() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, true); + col.addSymbol("server1"); + buffer.nextRow(); + + col.addSymbol(null); // Null symbol + buffer.nextRow(); + + col.addSymbol("server2"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + } + + // ==================== MULTIPLE ROWS TESTS ==================== + + @Test + public void testEncodeSingleRowWithBoolean() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + col.addBoolean(true); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeSingleRowWithDouble() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + col.addDouble(23.5); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== MIXED COLUMN TYPES ==================== + + @Test + public void testEncodeSingleRowWithLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Add a long column + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("value", TYPE_LONG, false); + col.addLong(12345L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); // At least header size + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify header magic + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + + // Version + Assert.assertEquals(VERSION_1, Unsafe.getUnsafe().getByte(ptr + 4)); + + // Table count (little-endian short) + Assert.assertEquals((short) 1, Unsafe.getUnsafe().getShort(ptr + 6)); + } + } + + @Test + public void testEncodeSingleRowWithString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== EDGE CASES ==================== + + @Test + public void testEncodeSingleRowWithTimestamp() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Add a timestamp column (designated timestamp uses empty name) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); // Micros + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeSingleSymbol() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeSpecialDoubles() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.MAX_VALUE); + buffer.nextRow(); + + col.addDouble(Double.MIN_VALUE); + buffer.nextRow(); + + col.addDouble(Double.POSITIVE_INFINITY); + buffer.nextRow(); + + col.addDouble(Double.NEGATIVE_INFINITY); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + } + + @Test + public void testEncodeSymbolWithManyDistinctValues() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + for (int i = 0; i < 100; i++) { + col.addSymbol("server" + i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + } + + @Test + public void testEncodeUnicodeString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("Hello 世界 🌍"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeUuid() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + col.addUuid(0x123456789ABCDEF0L, 0xFEDCBA9876543210L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeWithDeltaDict_freshConnection_sendsAllSymbols() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Add symbol column with global IDs + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + + // Simulate adding symbols via global dictionary + int id1 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id2 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + col.addSymbolWithGlobalId("AAPL", id1); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id2); + buffer.nextRow(); + + // Fresh connection: confirmedMaxId = -1, so delta should include all symbols (0, 1) + int confirmedMaxId = -1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify header flag has FLAG_DELTA_SYMBOL_DICT set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + } + + @Test + public void testEncodeWithDeltaDict_noNewSymbols_sendsEmptyDelta() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Pre-populate dictionary with all symbols + int id0 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id1 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Use only existing symbols + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", id0); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id1); + buffer.nextRow(); + + // Server has confirmed all symbols (0-1), batchMaxId is 1 + int confirmedMaxId = 1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + pos++; + + // Read deltaCount varint (should be 0) + int deltaCount = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(0, deltaCount); + } + } + + @Test + public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Pre-populate dictionary (simulating symbols already sent) + globalDict.getOrAddSymbol("AAPL"); // ID 0 + globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Now add new symbols + int id2 = globalDict.getOrAddSymbol("MSFT"); // ID 2 + int id3 = globalDict.getOrAddSymbol("TSLA"); // ID 3 + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("MSFT", id2); + buffer.nextRow(); + col.addSymbolWithGlobalId("TSLA", id3); + buffer.nextRow(); + + // Server has confirmed IDs 0-1, so delta should only include 2-3 + int confirmedMaxId = 1; + int batchMaxId = 3; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + } + } + + // ==================== SCHEMA REFERENCE TESTS ==================== + + @Test + public void testEncodeWithSchemaRef() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); + + int size = encoder.encode(buffer, true); // Use schema reference + Assert.assertTrue(size > 12); + } + } + + // ==================== BUFFER REUSE TESTS ==================== + + @Test + public void testEncodeZeroLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(0L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncoderReusability() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + // Encode first message + QwpTableBuffer buffer1 = new QwpTableBuffer("table1"); + QwpTableBuffer.ColumnBuffer col1 = buffer1.getOrCreateColumn("x", TYPE_LONG, false); + col1.addLong(1L); + buffer1.nextRow(); + int size1 = encoder.encode(buffer1, false); + + // Encode second message (encoder should reset internally) + QwpTableBuffer buffer2 = new QwpTableBuffer("table2"); + QwpTableBuffer.ColumnBuffer col2 = buffer2.getOrCreateColumn("y", TYPE_DOUBLE, false); + col2.addDouble(2.0); + buffer2.nextRow(); + int size2 = encoder.encode(buffer2, false); + + // Both should succeed + Assert.assertTrue(size1 > 12); + Assert.assertTrue(size2 > 12); + } + } + + // ==================== ALL BASIC TYPES IN ONE ROW ==================== + + @Test + public void testGlobalSymbolDictionaryBasics() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Test sequential IDs + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + Assert.assertEquals(2, dict.getOrAddSymbol("MSFT")); + + // Test deduplication + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + + // Test retrieval + Assert.assertEquals("AAPL", dict.getSymbol(0)); + Assert.assertEquals("GOOG", dict.getSymbol(1)); + Assert.assertEquals("MSFT", dict.getSymbol(2)); + + // Test size + Assert.assertEquals(3, dict.size()); + } + + // ==================== Delta Symbol Dictionary Tests ==================== + + @Test + public void testGorillaEncoding_compressionRatio() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("metrics"); + + // Add many timestamps with constant delta - best case for Gorilla + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Calculate theoretical minimum size for Gorilla: + // - Header: 12 bytes + // - Table header, column schema, etc. + // - First timestamp: 8 bytes + // - Second timestamp: 8 bytes + // - Remaining 998 timestamps: 998 bits (1 bit each for DoD=0) = ~125 bytes + + // Calculate size without Gorilla (1000 * 8 = 8000 bytes just for timestamps) + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // For constant delta, Gorilla should achieve significant compression + double compressionRatio = (double) sizeWithGorilla / sizeWithoutGorilla; + Assert.assertTrue("Compression ratio should be < 0.2 for constant delta", + compressionRatio < 0.2); + } + } + + @Test + public void testGorillaEncoding_multipleTimestampColumns() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Add multiple timestamp columns + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); + + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); + + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Compare with uncompressed + encoder.setGorillaEnabled(false); + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); + + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); + + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + Assert.assertTrue("Gorilla should compress multiple timestamp columns", + sizeWithGorilla < sizeWithoutGorilla); + } + } + + @Test + public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Add multiple timestamps with constant delta (best compression) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Now encode without Gorilla + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // Gorilla should produce smaller output for constant-delta timestamps + Assert.assertTrue("Gorilla encoding should be smaller", + sizeWithGorilla < sizeWithoutGorilla); + } + } + + @Test + public void testGorillaEncoding_nanosTimestamps() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Use TYPE_TIMESTAMP_NANOS + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP_NANOS, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000000000000L + i * 1000000L); // Nanos with millisecond intervals + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testGorillaEncoding_singleTimestamp_usesUncompressed() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Single timestamp - should use uncompressed + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testGorillaEncoding_twoTimestamps_usesUncompressed() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Only 2 timestamps - should use uncompressed (Gorilla needs 3+) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + col.addLong(2000000L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testGorillaEncoding_varyingDelta() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Varying deltas that exercise different buckets + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + long[] timestamps = { + 1000000000L, + 1000001000L, // delta=1000 + 1000002000L, // DoD=0 + 1000003050L, // DoD=50 + 1000004200L, // DoD=100 + 1000006200L, // DoD=850 + }; + + for (long ts : timestamps) { + col.addLong(ts); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testGorillaFlagDisabled() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(false); + Assert.assertFalse(encoder.isGorillaEnabled()); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + encoder.encode(buffer, false); + + // Check flags byte doesn't have Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(0, flags & FLAG_GORILLA); + } + } + + @Test + public void testGorillaFlagEnabled() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + Assert.assertTrue(encoder.isGorillaEnabled()); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + encoder.encode(buffer, false); + + // Check flags byte has Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testPayloadLengthPatched() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + + // Payload length is at offset 8 (4 magic + 1 version + 1 flags + 2 tablecount) + QwpBufferWriter buf = encoder.getBuffer(); + int payloadLength = Unsafe.getUnsafe().getInt(buf.getBufferPtr() + 8); + + // Payload length should be total size minus header (12 bytes) + Assert.assertEquals(size - 12, payloadLength); + } + } + + @Test + public void testReset() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); + + int size1 = encoder.encode(buffer, false); + + // Reset and encode again + buffer.reset(); + col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(2L); + buffer.nextRow(); + + int size2 = encoder.encode(buffer, false); + + // Sizes should be similar (same schema) + Assert.assertEquals(size1, size2); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java new file mode 100644 index 0000000..a128ed3 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -0,0 +1,458 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.network.PlainSocketFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +/** + * Unit tests for QwpWebSocketSender. + * These tests focus on state management and API validation without requiring a live server. + */ +public class QwpWebSocketSenderTest { + + @Test + public void testAtAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testAtInstantAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.at(Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testAtNowAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testBoolColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.boolColumn("x", true); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testBufferViewNotSupported() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.bufferView(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("not supported")); + } + } + + @Test + public void testCancelRowAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.cancelRow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testCloseIdemponent() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + sender.close(); // Should not throw + } + + @Test + public void testConnectToClosedPort() { + try { + QwpWebSocketSender.connect("127.0.0.1", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Failed to connect")); + } + } + + @Test + public void testDoubleArrayAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.doubleArray("x", new double[]{1.0, 2.0}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testDoubleColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.doubleColumn("x", 1.0); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testGorillaEnabledByDefault() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + Assert.assertTrue(sender.isGorillaEnabled()); + } + } + + @Test + public void testLongArrayAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.longArray("x", new long[]{1L, 2L}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testLongColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testNullArrayReturnsThis() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + // Null arrays should be no-ops and return sender + Assert.assertSame(sender, sender.doubleArray("x", (double[]) null)); + Assert.assertSame(sender, sender.longArray("x", (long[]) null)); + } + } + + @Test + public void testOperationsAfterCloseThrow() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.table("test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testResetAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.reset(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testSealAndSwapRollsBackOnEnqueueFailure() throws Exception { + try (QwpWebSocketSender sender = createUnconnectedAsyncSender(); ThrowingOnceWebSocketSendQueue queue = new ThrowingOnceWebSocketSendQueue()) { + setSendQueue(sender, queue); + + MicrobatchBuffer originalActive = getActiveBuffer(sender); + originalActive.writeByte((byte) 7); + originalActive.incrementRowCount(); + + try { + invokeSealAndSwapBuffer(sender); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Synthetic enqueue failure")); + } + + // Failed enqueue must not strand the sealed buffer. + Assert.assertSame(originalActive, getActiveBuffer(sender)); + Assert.assertTrue(originalActive.isFilling()); + Assert.assertTrue(originalActive.hasData()); + Assert.assertEquals(1, originalActive.getRowCount()); + + // Retry should be possible on the same sender instance. + invokeSealAndSwapBuffer(sender); + Assert.assertNotSame(originalActive, getActiveBuffer(sender)); + } + } + + @Test + public void testSetGorillaEnabled() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.setGorillaEnabled(false); + Assert.assertFalse(sender.isGorillaEnabled()); + sender.setGorillaEnabled(true); + Assert.assertTrue(sender.isGorillaEnabled()); + } + } + + @Test + public void testStringColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.stringColumn("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testSymbolAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.symbol("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testTableBeforeAtNowRequired() { + try { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + } + + @Test + public void testTableBeforeAtRequired() { + try { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + } + + @Test + public void testTableBeforeColumnsRequired() { + // Create sender without connecting (we'll catch the error earlier) + try { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + } + + @Test + public void testTimestampColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.timestampColumn("x", 1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testTimestampColumnInstantAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.timestampColumn("x", Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + private static MicrobatchBuffer getActiveBuffer(QwpWebSocketSender sender) throws Exception { + Field field = QwpWebSocketSender.class.getDeclaredField("activeBuffer"); + field.setAccessible(true); + return (MicrobatchBuffer) field.get(sender); + } + + private static void invokeSealAndSwapBuffer(QwpWebSocketSender sender) throws Exception { + Method method = QwpWebSocketSender.class.getDeclaredMethod("sealAndSwapBuffer"); + method.setAccessible(true); + try { + method.invoke(sender); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new RuntimeException(cause); + } + } + + private static void setSendQueue(QwpWebSocketSender sender, WebSocketSendQueue queue) throws Exception { + Field field = QwpWebSocketSender.class.getDeclaredField("sendQueue"); + field.setAccessible(true); + field.set(sender, queue); + } + + /** + * Creates an async sender without connecting. + */ + private QwpWebSocketSender createUnconnectedAsyncSender() { + return QwpWebSocketSender.createForTesting("localhost", 9000, + 500, 0, 0L, // autoFlushRows, autoFlushBytes, autoFlushIntervalNanos + 8); // inFlightWindowSize + } + + /** + * Creates an async sender with custom flow control settings without connecting. + */ + private QwpWebSocketSender createUnconnectedAsyncSenderWithFlowControl( + int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, + int inFlightWindowSize) { + return QwpWebSocketSender.createForTesting("localhost", 9000, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize); + } + + /** + * Creates a sender without connecting. + * For unit tests that don't need actual connectivity. + */ + private QwpWebSocketSender createUnconnectedSender() { + return QwpWebSocketSender.createForTesting("localhost", 9000, 1); // window=1 for sync + } + + private static class NoOpWebSocketClient extends WebSocketClient { + private NoOpWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean isConnected() { + return false; + } + + @Override + public void sendBinary(long dataPtr, int length) { + // no-op + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + private static class ThrowingOnceWebSocketSendQueue extends WebSocketSendQueue { + private boolean failOnce = true; + + private ThrowingOnceWebSocketSendQueue() { + super(new NoOpWebSocketClient(), null, 50, 50); + } + + @Override + public boolean enqueue(MicrobatchBuffer buffer) { + if (failOnce) { + failOnce = false; + throw new LineSenderException("Synthetic enqueue failure"); + } + return true; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java new file mode 100644 index 0000000..4629272 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java @@ -0,0 +1,339 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +public class WebSocketSendQueueTest { + + @Test + public void testEnqueueTimeoutWhenPendingSlotOccupied() { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; + + try { + // Keep window full so I/O thread cannot drain pending slot. + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 100, 500); + queue.enqueue(batch0); + + try { + queue.enqueue(batch1); + fail("Expected enqueue timeout"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Enqueue timeout")); + } + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + } + + @Test + public void testEnqueueWaitsUntilSlotAvailable() throws Exception { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; + + try { + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 2_000, 500); + final WebSocketSendQueue finalQueue = queue; + queue.enqueue(batch0); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + AtomicReference errorRef = new AtomicReference<>(); + + Thread t = new Thread(() -> { + started.countDown(); + try { + finalQueue.enqueue(batch1); + } catch (Throwable t1) { + errorRef.set(t1); + } finally { + finished.countDown(); + } + }); + t.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertEquals("Second enqueue should still be waiting", 1, finished.getCount()); + + // Free space so I/O thread can poll pending slot. + window.acknowledgeUpTo(0); + + assertTrue("Second enqueue should complete", finished.await(2, TimeUnit.SECONDS)); + assertNull(errorRef.get()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + } + + @Test + public void testFlushFailsOnInvalidAckPayload() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch payloadDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + emitBinary(handler, new byte[]{1, 2, 3}); + payloadDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected invalid payload callback", payloadDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure on invalid payload"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Invalid ACK response payload")); + } + } finally { + closeQuietly(queue); + client.close(); + } + } + + @Test + public void testFlushFailsOnReceiveIoError() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch receiveAttempted = new CountDownLatch(1); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + receiveAttempted.countDown(); + throw new RuntimeException("recv-fail"); + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected receive attempt", receiveAttempted.await(2, TimeUnit.SECONDS)); + long deadline = System.currentTimeMillis() + 2_000; + while (queue.getLastError() == null && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + assertNotNull("Expected queue error after receive failure", queue.getLastError()); + + try { + queue.flush(); + fail("Expected flush failure after receive error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Error receiving response")); + } + } finally { + closeQuietly(queue); + client.close(); + } + } + + @Test + public void testFlushFailsOnSendIoError() { + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch = sealedBuffer((byte) 42); + WebSocketSendQueue queue = null; + + try { + client.setSendBehavior((dataPtr, length) -> { + throw new RuntimeException("send-fail"); + }); + queue = new WebSocketSendQueue(client, null, 1_000, 500); + queue.enqueue(batch); + + try { + queue.flush(); + fail("Expected flush failure after send error"); + } catch (LineSenderException e) { + assertTrue( + e.getMessage().contains("Error sending batch") + || e.getMessage().contains("Error in send queue I/O thread") + ); + } + } finally { + closeQuietly(queue); + batch.close(); + client.close(); + } + } + + @Test + public void testFlushFailsWhenServerClosesConnection() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch closeDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + handler.onClose(1006, "boom"); + closeDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected close callback", closeDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure after close"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("closed")); + } + } finally { + closeQuietly(queue); + client.close(); + } + } + + private static void closeQuietly(WebSocketSendQueue queue) { + if (queue != null) { + queue.close(); + } + } + + private static void emitBinary(WebSocketFrameHandler handler, byte[] payload) { + long ptr = Unsafe.malloc(payload.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < payload.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, payload[i]); + } + handler.onBinaryMessage(ptr, payload.length); + } finally { + Unsafe.free(ptr, payload.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static MicrobatchBuffer sealedBuffer(byte value) { + MicrobatchBuffer buffer = new MicrobatchBuffer(64); + buffer.writeByte(value); + buffer.incrementRowCount(); + buffer.seal(); + return buffer; + } + + private interface SendBehavior { + void send(long dataPtr, int length); + } + + private interface TryReceiveBehavior { + boolean tryReceive(WebSocketFrameHandler handler); + } + + private static class FakeWebSocketClient extends WebSocketClient { + private volatile TryReceiveBehavior behavior = handler -> false; + private volatile boolean connected = true; + private volatile SendBehavior sendBehavior = (dataPtr, length) -> { + }; + + private FakeWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void close() { + connected = false; + super.close(); + } + + @Override + public boolean isConnected() { + return connected; + } + + @Override + public void sendBinary(long dataPtr, int length) { + sendBehavior.send(dataPtr, length); + } + + public void setSendBehavior(SendBehavior sendBehavior) { + this.sendBehavior = sendBehavior; + } + + public void setTryReceiveBehavior(TryReceiveBehavior behavior) { + this.behavior = behavior; + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + return behavior.tryReceive(handler); + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} From 233f4ed71d6fbe26e996bf37ac222e69a539c43b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 15:57:39 +0100 Subject: [PATCH 078/230] Port 5 QWP protocol tests from core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add client-side test coverage for QwpVarint (16 tests), QwpZigZag (10 tests), QwpConstants (14 tests), QwpNullBitmap (13 tests), and QwpSchemaHash (20 tests). Adaptations from core originals: - MemoryTag.NATIVE_DEFAULT → NATIVE_ILP_RSS - QwpParseException → IllegalArgumentException - QwpConstants tests cover newer type codes (0x10–0x16) and FLAG_DELTA_SYMBOL_DICT Co-Authored-By: Claude Opus 4.6 --- .../qwp/protocol/QwpConstantsTest.java | 232 +++++++++++++ .../qwp/protocol/QwpNullBitmapTest.java | 289 +++++++++++++++ .../qwp/protocol/QwpSchemaHashTest.java | 328 ++++++++++++++++++ .../cutlass/qwp/protocol/QwpVarintTest.java | 262 ++++++++++++++ .../cutlass/qwp/protocol/QwpZigZagTest.java | 166 +++++++++ 5 files changed, 1277 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java new file mode 100644 index 0000000..2b6ce04 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -0,0 +1,232 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +public class QwpConstantsTest { + + @Test + public void testDefaultLimits() { + Assert.assertEquals(16 * 1024 * 1024, DEFAULT_MAX_BATCH_SIZE); + Assert.assertEquals(256, DEFAULT_MAX_TABLES_PER_BATCH); + Assert.assertEquals(1_000_000, DEFAULT_MAX_ROWS_PER_TABLE); + Assert.assertEquals(2048, MAX_COLUMNS_PER_TABLE); + Assert.assertEquals(64 * 1024, DEFAULT_INITIAL_RECV_BUFFER_SIZE); + Assert.assertEquals(4, DEFAULT_MAX_IN_FLIGHT_BATCHES); + } + + @Test + public void testFlagBitPositions() { + // Verify flag bits are at correct positions + Assert.assertEquals(0x01, FLAG_LZ4); + Assert.assertEquals(0x02, FLAG_ZSTD); + Assert.assertEquals(0x04, FLAG_GORILLA); + Assert.assertEquals(0x03, FLAG_COMPRESSION_MASK); + Assert.assertEquals(0x08, FLAG_DELTA_SYMBOL_DICT); + } + + @Test + public void testGetFixedTypeSize() { + Assert.assertEquals(0, QwpConstants.getFixedTypeSize(TYPE_BOOLEAN)); // Bit-packed + Assert.assertEquals(1, QwpConstants.getFixedTypeSize(TYPE_BYTE)); + Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_SHORT)); + Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_CHAR)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_INT)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_LONG)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_FLOAT)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DOUBLE)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_TIMESTAMP)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_TIMESTAMP_NANOS)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DATE)); + Assert.assertEquals(16, QwpConstants.getFixedTypeSize(TYPE_UUID)); + Assert.assertEquals(32, QwpConstants.getFixedTypeSize(TYPE_LONG256)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DECIMAL64)); + Assert.assertEquals(16, QwpConstants.getFixedTypeSize(TYPE_DECIMAL128)); + Assert.assertEquals(32, QwpConstants.getFixedTypeSize(TYPE_DECIMAL256)); + + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_STRING)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_SYMBOL)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_DOUBLE_ARRAY)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_LONG_ARRAY)); + } + + @Test + public void testGetTypeName() { + Assert.assertEquals("BOOLEAN", QwpConstants.getTypeName(TYPE_BOOLEAN)); + Assert.assertEquals("INT", QwpConstants.getTypeName(TYPE_INT)); + Assert.assertEquals("STRING", QwpConstants.getTypeName(TYPE_STRING)); + Assert.assertEquals("TIMESTAMP", QwpConstants.getTypeName(TYPE_TIMESTAMP)); + Assert.assertEquals("TIMESTAMP_NANOS", QwpConstants.getTypeName(TYPE_TIMESTAMP_NANOS)); + Assert.assertEquals("DOUBLE_ARRAY", QwpConstants.getTypeName(TYPE_DOUBLE_ARRAY)); + Assert.assertEquals("LONG_ARRAY", QwpConstants.getTypeName(TYPE_LONG_ARRAY)); + Assert.assertEquals("DECIMAL64", QwpConstants.getTypeName(TYPE_DECIMAL64)); + Assert.assertEquals("DECIMAL128", QwpConstants.getTypeName(TYPE_DECIMAL128)); + Assert.assertEquals("DECIMAL256", QwpConstants.getTypeName(TYPE_DECIMAL256)); + Assert.assertEquals("CHAR", QwpConstants.getTypeName(TYPE_CHAR)); + + // Test nullable types + byte nullableInt = (byte) (TYPE_INT | TYPE_NULLABLE_FLAG); + Assert.assertEquals("INT?", QwpConstants.getTypeName(nullableInt)); + + byte nullableString = (byte) (TYPE_STRING | TYPE_NULLABLE_FLAG); + Assert.assertEquals("STRING?", QwpConstants.getTypeName(nullableString)); + } + + @Test + public void testHeaderSize() { + Assert.assertEquals(12, HEADER_SIZE); + } + + @Test + public void testIsFixedWidthType() { + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_BOOLEAN)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_BYTE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_SHORT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_CHAR)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_INT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_FLOAT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DOUBLE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_TIMESTAMP)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_TIMESTAMP_NANOS)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DATE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_UUID)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG256)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL64)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL128)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL256)); + + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_STRING)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_SYMBOL)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_GEOHASH)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_VARCHAR)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_DOUBLE_ARRAY)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_LONG_ARRAY)); + } + + @Test + public void testMagicBytesCapabilityRequest() { + // "ILP?" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '?'}; + Assert.assertEquals((byte) (MAGIC_CAPABILITY_REQUEST & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesCapabilityResponse() { + // "ILP!" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '!'}; + Assert.assertEquals((byte) (MAGIC_CAPABILITY_RESPONSE & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesFallback() { + // "ILP0" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '0'}; + Assert.assertEquals((byte) (MAGIC_FALLBACK & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesValue() { + // "ILP4" in ASCII: I=0x49, L=0x4C, P=0x50, 4=0x34 + // Little-endian: 0x34504C49 + Assert.assertEquals(0x34504C49, MAGIC_MESSAGE); + + // Verify ASCII encoding + byte[] expected = new byte[]{'I', 'L', 'P', '4'}; + Assert.assertEquals((byte) (MAGIC_MESSAGE & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 24) & 0xFF), expected[3]); + } + + @Test + public void testNullableFlag() { + Assert.assertEquals((byte) 0x80, TYPE_NULLABLE_FLAG); + Assert.assertEquals(0x7F, TYPE_MASK); + + // Test nullable type extraction + byte nullableInt = (byte) (TYPE_INT | TYPE_NULLABLE_FLAG); + Assert.assertEquals(TYPE_INT, nullableInt & TYPE_MASK); + } + + @Test + public void testSchemaModes() { + Assert.assertEquals(0x00, SCHEMA_MODE_FULL); + Assert.assertEquals(0x01, SCHEMA_MODE_REFERENCE); + } + + @Test + public void testStatusCodes() { + Assert.assertEquals(0x00, STATUS_OK); + Assert.assertEquals(0x01, STATUS_PARTIAL); + Assert.assertEquals(0x02, STATUS_SCHEMA_REQUIRED); + Assert.assertEquals(0x03, STATUS_SCHEMA_MISMATCH); + Assert.assertEquals(0x04, STATUS_TABLE_NOT_FOUND); + Assert.assertEquals(0x05, STATUS_PARSE_ERROR); + Assert.assertEquals(0x06, STATUS_INTERNAL_ERROR); + Assert.assertEquals(0x07, STATUS_OVERLOADED); + } + + @Test + public void testTypeCodes() { + // Verify type codes match specification + Assert.assertEquals(0x01, TYPE_BOOLEAN); + Assert.assertEquals(0x02, TYPE_BYTE); + Assert.assertEquals(0x03, TYPE_SHORT); + Assert.assertEquals(0x04, TYPE_INT); + Assert.assertEquals(0x05, TYPE_LONG); + Assert.assertEquals(0x06, TYPE_FLOAT); + Assert.assertEquals(0x07, TYPE_DOUBLE); + Assert.assertEquals(0x08, TYPE_STRING); + Assert.assertEquals(0x09, TYPE_SYMBOL); + Assert.assertEquals(0x0A, TYPE_TIMESTAMP); + Assert.assertEquals(0x0B, TYPE_DATE); + Assert.assertEquals(0x0C, TYPE_UUID); + Assert.assertEquals(0x0D, TYPE_LONG256); + Assert.assertEquals(0x0E, TYPE_GEOHASH); + Assert.assertEquals(0x0F, TYPE_VARCHAR); + Assert.assertEquals(0x10, TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0x11, TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0x12, TYPE_LONG_ARRAY); + Assert.assertEquals(0x13, TYPE_DECIMAL64); + Assert.assertEquals(0x14, TYPE_DECIMAL128); + Assert.assertEquals(0x15, TYPE_DECIMAL256); + Assert.assertEquals(0x16, TYPE_CHAR); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java new file mode 100644 index 0000000..086d24f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java @@ -0,0 +1,289 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpNullBitmap; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +public class QwpNullBitmapTest { + + @Test + public void testAllNulls() { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); + + for (int i = 0; i < rowCount; i++) { + Assert.assertTrue(QwpNullBitmap.isNull(address, i)); + } + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testAllNullsPartialByte() { + // Test with row count not divisible by 8 + int rowCount = 10; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testBitmapBitOrder() { + // Test LSB-first bit ordering + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set bit 0 (LSB) + QwpNullBitmap.setNull(address, 0); + byte b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b00000001, b & 0xFF); + + // Set bit 7 (MSB of first byte) + QwpNullBitmap.setNull(address, 7); + b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b10000001, b & 0xFF); + + // Set bit 3 + QwpNullBitmap.setNull(address, 3); + b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b10001001, b & 0xFF); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testBitmapByteAlignment() { + // Test that bits 8-15 go into second byte + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set bit 8 (first bit of second byte) + QwpNullBitmap.setNull(address, 8); + Assert.assertEquals(0, Unsafe.getUnsafe().getByte(address) & 0xFF); + Assert.assertEquals(0b00000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + + // Set bit 15 (last bit of second byte) + QwpNullBitmap.setNull(address, 15); + Assert.assertEquals(0b10000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testBitmapSizeCalculation() { + Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(1)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(7)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(8)); + Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(9)); + Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(16)); + Assert.assertEquals(3, QwpNullBitmap.sizeInBytes(17)); + Assert.assertEquals(125, QwpNullBitmap.sizeInBytes(1000)); + Assert.assertEquals(125000, QwpNullBitmap.sizeInBytes(1000000)); + } + + @Test + public void testBitmapWithPartialLastByte() { + // 10 rows = 2 bytes, but only 2 bits used in second byte + int rowCount = 10; + int size = QwpNullBitmap.sizeInBytes(rowCount); + Assert.assertEquals(2, size); + + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set row 9 (bit 1 of second byte) + QwpNullBitmap.setNull(address, 9); + Assert.assertTrue(QwpNullBitmap.isNull(address, 9)); + Assert.assertEquals(0b00000010, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + + Assert.assertEquals(1, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testByteArrayOperations() { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + byte[] bitmap = new byte[size]; + int offset = 0; + + QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); + + QwpNullBitmap.setNull(bitmap, offset, 0); + QwpNullBitmap.setNull(bitmap, offset, 5); + QwpNullBitmap.setNull(bitmap, offset, 15); + + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 0)); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 1)); + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 5)); + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 15)); + + Assert.assertEquals(3, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); + + QwpNullBitmap.clearNull(bitmap, offset, 5); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 5)); + Assert.assertEquals(2, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); + } + + @Test + public void testByteArrayWithOffset() { + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + byte[] bitmap = new byte[10 + size]; // Extra padding + int offset = 5; // Start at offset 5 + + QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); + QwpNullBitmap.setNull(bitmap, offset, 3); + + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 3)); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 4)); + } + + @Test + public void testClearNull() { + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.isNull(address, 3)); + + QwpNullBitmap.clearNull(address, 3); + Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); + Assert.assertEquals(7, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEmptyBitmap() { + Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); + } + + @Test + public void testLargeBitmap() { + int rowCount = 100000; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set every 100th row as null + int expectedNulls = 0; + for (int i = 0; i < rowCount; i += 100) { + QwpNullBitmap.setNull(address, i); + expectedNulls++; + } + + Assert.assertEquals(expectedNulls, QwpNullBitmap.countNulls(address, rowCount)); + + // Verify some random positions + Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 100)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 99900)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 99)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testMixedNulls() { + int rowCount = 20; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set specific rows as null: 0, 2, 5, 19 + QwpNullBitmap.setNull(address, 0); + QwpNullBitmap.setNull(address, 2); + QwpNullBitmap.setNull(address, 5); + QwpNullBitmap.setNull(address, 19); + + Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 2)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 4)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 5)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 19)); + + Assert.assertEquals(4, QwpNullBitmap.countNulls(address, rowCount)); + Assert.assertFalse(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertFalse(QwpNullBitmap.noneNull(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testNoNulls() { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.noneNull(address, rowCount)); + Assert.assertEquals(0, QwpNullBitmap.countNulls(address, rowCount)); + + for (int i = 0; i < rowCount; i++) { + Assert.assertFalse(QwpNullBitmap.isNull(address, i)); + } + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java new file mode 100644 index 0000000..75cae68 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java @@ -0,0 +1,328 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaHash; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +public class QwpSchemaHashTest { + + private static final byte TYPE_LONG = 0x05; + + @Test + public void testColumnOrderMatters() { + // Order 1 + String[] names1 = {"price", "symbol"}; + byte[] types1 = {0x07, 0x09}; + + // Order 2 (different order) + String[] names2 = {"symbol", "price"}; + byte[] types2 = {0x09, 0x07}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names1, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names2, types2); + + Assert.assertNotEquals("Column order should affect hash", hash1, hash2); + } + + @Test + public void testDeterministic() { + String[] names = {"col1", "col2", "col3"}; + byte[] types = {0x01, 0x02, 0x03}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + long hash3 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("Hash should be deterministic", hash1, hash2); + Assert.assertEquals("Hash should be deterministic", hash2, hash3); + } + + @Test + public void testEmptySchema() { + String[] names = {}; + byte[] types = {}; + long hash = QwpSchemaHash.computeSchemaHash(names, types); + // Empty input should produce the same hash consistently + Assert.assertEquals(hash, QwpSchemaHash.computeSchemaHash(names, types)); + } + + @Test + public void testHasherReset() { + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + + byte[] data1 = "first".getBytes(StandardCharsets.UTF_8); + byte[] data2 = "second".getBytes(StandardCharsets.UTF_8); + + // Hash first data + hasher.reset(0); + hasher.update(data1); + long hash1 = hasher.getValue(); + + // Reset and hash second data + hasher.reset(0); + hasher.update(data2); + long hash2 = hasher.getValue(); + + // Should be different + Assert.assertNotEquals(hash1, hash2); + + // Reset and hash first again - should be same as original + hasher.reset(0); + hasher.update(data1); + Assert.assertEquals(hash1, hasher.getValue()); + } + + @Test + public void testHasherStreaming() { + // Test that streaming hasher produces same result as one-shot + byte[] data = "streaming test data for the hasher".getBytes(StandardCharsets.UTF_8); + + // One-shot + long oneShot = QwpSchemaHash.hash(data); + + // Streaming - byte by byte + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + hasher.reset(0); + for (byte b : data) { + hasher.update(b); + } + long streaming = hasher.getValue(); + + Assert.assertEquals("Streaming should match one-shot", oneShot, streaming); + } + + @Test + public void testHasherStreamingChunks() { + // Test streaming with various chunk sizes + byte[] data = "This is a longer test string to verify chunked hashing works correctly!".getBytes(StandardCharsets.UTF_8); + + long oneShot = QwpSchemaHash.hash(data); + + // Streaming - in chunks + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + hasher.reset(0); + + int pos = 0; + int[] chunkSizes = {5, 10, 3, 20, 7, 15}; + for (int chunkSize : chunkSizes) { + int toAdd = Math.min(chunkSize, data.length - pos); + if (toAdd > 0) { + hasher.update(data, pos, toAdd); + pos += toAdd; + } + } + // Add remaining + if (pos < data.length) { + hasher.update(data, pos, data.length - pos); + } + + Assert.assertEquals("Chunked streaming should match one-shot", oneShot, hasher.getValue()); + } + + @Test + public void testLargeSchema() { + // Test with many columns + int columnCount = 100; + String[] names = new String[columnCount]; + byte[] types = new byte[columnCount]; + + for (int i = 0; i < columnCount; i++) { + names[i] = "column_" + i; + types[i] = (byte) ((i % 15) + 1); // Cycle through types 1-15 + } + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("Large schema should hash consistently", hash1, hash2); + } + + @Test + public void testMultipleColumns() { + String[] names = {"symbol", "price", "timestamp"}; + byte[] types = {0x09, 0x07, 0x0A}; // SYMBOL, DOUBLE, TIMESTAMP + long hash = QwpSchemaHash.computeSchemaHash(names, types); + Assert.assertNotEquals(0, hash); + } + + @Test + public void testNameAffectsHash() { + // Different names, same type + byte[] types = {0x07}; // DOUBLE + + String[] names1 = {"price"}; + String[] names2 = {"value"}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names1, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names2, types); + + Assert.assertNotEquals("Name should affect hash", hash1, hash2); + } + + @Test + public void testNullableFlagAffectsHash() { + String[] names = {"value"}; + + // Non-nullable + byte[] types1 = {0x05}; // LONG + // Nullable (high bit set) + byte[] types2 = {(byte) 0x85}; // LONG | 0x80 + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types2); + + Assert.assertNotEquals("Nullable flag should affect hash", hash1, hash2); + } + + @Test + public void testSchemaHashWithUtf8Names() { + // Test UTF-8 column names + String[] names = {"prix", "日時", "価格"}; // French, Japanese for datetime, Japanese for price + byte[] types = {0x07, 0x0A, 0x07}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("UTF-8 names should hash consistently", hash1, hash2); + Assert.assertNotEquals(0, hash1); + } + + @Test + public void testSingleColumn() { + String[] names = {"price"}; + byte[] types = {0x07}; // DOUBLE + long hash = QwpSchemaHash.computeSchemaHash(names, types); + Assert.assertNotEquals(0, hash); + } + + @Test + public void testTypeAffectsHash() { + // Same name, different type + String[] names = {"value"}; + + byte[] types1 = {0x04}; // INT + byte[] types2 = {0x05}; // LONG + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types2); + + Assert.assertNotEquals("Type should affect hash", hash1, hash2); + } + + @Test + public void testXXHash64DirectMemory() { + byte[] data = "test data".getBytes(StandardCharsets.UTF_8); + long addr = Unsafe.malloc(data.length, MemoryTag.NATIVE_ILP_RSS); + try { + for (int i = 0; i < data.length; i++) { + Unsafe.getUnsafe().putByte(addr + i, data[i]); + } + + long hashFromBytes = QwpSchemaHash.hash(data); + long hashFromMem = QwpSchemaHash.hash(addr, data.length); + + Assert.assertEquals("Direct memory hash should match byte array hash", hashFromBytes, hashFromMem); + } finally { + Unsafe.free(addr, data.length, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testXXHash64Empty() { + byte[] data = new byte[0]; + long hash = QwpSchemaHash.hash(data); + // XXH64("", 0) = 0xEF46DB3751D8E999 + Assert.assertEquals(0xEF46DB3751D8E999L, hash); + } + + @Test + public void testXXHash64Exactly32Bytes() { + // Edge case: exactly 32 bytes + byte[] data = new byte[32]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i & 0xFF); + } + + long hash = QwpSchemaHash.hash(data); + Assert.assertNotEquals(0, hash); + Assert.assertEquals(hash, QwpSchemaHash.hash(data)); + } + + @Test + public void testXXHash64KnownValue() { + // Test against a known XXHash64 value + // "abc" with seed 0 should produce a specific value + byte[] data = "abc".getBytes(StandardCharsets.UTF_8); + long hash = QwpSchemaHash.hash(data); + + // XXH64("abc", 0) = 0x44BC2CF5AD770999 + Assert.assertEquals(0x44BC2CF5AD770999L, hash); + } + + @Test + public void testXXHash64LongerString() { + // Test with a longer string to exercise the main loop + byte[] data = "Hello, World! This is a test string for XXHash64.".getBytes(StandardCharsets.UTF_8); + long hash1 = QwpSchemaHash.hash(data); + long hash2 = QwpSchemaHash.hash(data); + Assert.assertEquals(hash1, hash2); + Assert.assertNotEquals(0, hash1); + } + + @Test + public void testXXHash64Over32Bytes() { + // Test data longer than 32 bytes to exercise the main processing loop + byte[] data = new byte[100]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i & 0xFF); + } + + long hash = QwpSchemaHash.hash(data); + Assert.assertNotEquals(0, hash); + + // Verify deterministic + Assert.assertEquals(hash, QwpSchemaHash.hash(data)); + } + + @Test + public void testXXHash64WithSeed() { + byte[] data = "test".getBytes(StandardCharsets.UTF_8); + + long hash0 = QwpSchemaHash.hash(data, 0, data.length, 0); + long hash1 = QwpSchemaHash.hash(data, 0, data.length, 1); + long hash42 = QwpSchemaHash.hash(data, 0, data.length, 42); + + // Different seeds should produce different hashes + Assert.assertNotEquals(hash0, hash1); + Assert.assertNotEquals(hash1, hash42); + Assert.assertNotEquals(hash0, hash42); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java new file mode 100644 index 0000000..558ecd2 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java @@ -0,0 +1,262 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpVarint; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Random; + +public class QwpVarintTest { + + @Test + public void testDecodeFromDirectMemory() { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + // Encode using byte array, decode from direct memory + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 300); + + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(addr + i, buf[i]); + } + + long decoded = QwpVarint.decode(addr, addr + len); + Assert.assertEquals(300, decoded); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testDecodeIncompleteVarint() { + // Byte with continuation bit set but no following byte + byte[] buf = new byte[]{(byte) 0x80}; + try { + QwpVarint.decode(buf, 0, 1); + Assert.fail("Should have thrown exception"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("incomplete varint")); + } + } + + @Test + public void testDecodeOverflow() { + // Create a buffer with too many continuation bytes (>10) + byte[] buf = new byte[12]; + for (int i = 0; i < 11; i++) { + buf[i] = (byte) 0x80; + } + buf[11] = 0x01; + + try { + QwpVarint.decode(buf, 0, 12); + Assert.fail("Should have thrown exception"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("varint overflow")); + } + } + + @Test + public void testDecodeResult() { + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 300); + + QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); + QwpVarint.decode(buf, 0, len, result); + + Assert.assertEquals(300, result.value); + Assert.assertEquals(len, result.bytesRead); + } + + @Test + public void testDecodeResultFromDirectMemory() { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + long endAddr = QwpVarint.encode(addr, 999999); + int expectedLen = (int) (endAddr - addr); + + QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); + QwpVarint.decode(addr, endAddr, result); + + Assert.assertEquals(999999, result.value); + Assert.assertEquals(expectedLen, result.bytesRead); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testDecodeResultReuse() { + byte[] buf = new byte[10]; + QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); + + // First decode + int len1 = QwpVarint.encode(buf, 0, 100); + QwpVarint.decode(buf, 0, len1, result); + Assert.assertEquals(100, result.value); + + // Reuse for second decode + result.reset(); + int len2 = QwpVarint.encode(buf, 0, 50000); + QwpVarint.decode(buf, 0, len2, result); + Assert.assertEquals(50000, result.value); + } + + @Test + public void testEncodeDecode127() { + // 127 is the maximum 1-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 127); + Assert.assertEquals(1, len); + Assert.assertEquals(0x7F, buf[0] & 0xFF); + Assert.assertEquals(127, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecode128() { + // 128 is the minimum 2-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 128); + Assert.assertEquals(2, len); + Assert.assertEquals(0x80, buf[0] & 0xFF); // 0 + continuation bit + Assert.assertEquals(0x01, buf[1] & 0xFF); // 1 + Assert.assertEquals(128, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecode16383() { + // 16383 (0x3FFF) is the maximum 2-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 16383); + Assert.assertEquals(2, len); + Assert.assertEquals(0xFF, buf[0] & 0xFF); // 127 + continuation bit + Assert.assertEquals(0x7F, buf[1] & 0xFF); // 127 + Assert.assertEquals(16383, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecode16384() { + // 16384 (0x4000) is the minimum 3-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 16384); + Assert.assertEquals(3, len); + Assert.assertEquals(16384, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecodeZero() { + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 0); + Assert.assertEquals(1, len); + Assert.assertEquals(0x00, buf[0] & 0xFF); + Assert.assertEquals(0, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeLargeValues() { + byte[] buf = new byte[10]; + + // Test various powers of 2 + long[] values = { + 1L << 20, // ~1M + 1L << 30, // ~1B + 1L << 40, // ~1T + 1L << 50, + 1L << 60, + Long.MAX_VALUE + }; + + for (long value : values) { + int len = QwpVarint.encode(buf, 0, value); + Assert.assertTrue(len > 0 && len <= 10); + Assert.assertEquals(value, QwpVarint.decode(buf, 0, len)); + } + } + + @Test + public void testEncodeSpecificValues() { + // Test values from the spec + byte[] buf = new byte[10]; + + // 300 = 0b100101100 + // Should encode as: 0xAC (0b10101100 = 44 + 128), 0x02 (0b00000010) + int len = QwpVarint.encode(buf, 0, 300); + Assert.assertEquals(2, len); + Assert.assertEquals(0xAC, buf[0] & 0xFF); + Assert.assertEquals(0x02, buf[1] & 0xFF); + + // Verify decode + Assert.assertEquals(300, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeToDirectMemory() { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + long endAddr = QwpVarint.encode(addr, 12345); + int len = (int) (endAddr - addr); + Assert.assertTrue(len > 0); + + // Read back and verify + long decoded = QwpVarint.decode(addr, endAddr); + Assert.assertEquals(12345, decoded); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodedLength() { + Assert.assertEquals(1, QwpVarint.encodedLength(0)); + Assert.assertEquals(1, QwpVarint.encodedLength(1)); + Assert.assertEquals(1, QwpVarint.encodedLength(127)); + Assert.assertEquals(2, QwpVarint.encodedLength(128)); + Assert.assertEquals(2, QwpVarint.encodedLength(16383)); + Assert.assertEquals(3, QwpVarint.encodedLength(16384)); + // Long.MAX_VALUE = 0x7FFFFFFFFFFFFFFF (63 bits) needs ceil(63/7) = 9 bytes + Assert.assertEquals(9, QwpVarint.encodedLength(Long.MAX_VALUE)); + // Test that actual encoding matches + byte[] buf = new byte[10]; + int actualLen = QwpVarint.encode(buf, 0, Long.MAX_VALUE); + Assert.assertEquals(actualLen, QwpVarint.encodedLength(Long.MAX_VALUE)); + } + + @Test + public void testRoundTripRandomValues() { + byte[] buf = new byte[10]; + Random random = new Random(42); // Fixed seed for reproducibility + + for (int i = 0; i < 1000; i++) { + long value = random.nextLong() & Long.MAX_VALUE; // Only positive values + int len = QwpVarint.encode(buf, 0, value); + long decoded = QwpVarint.decode(buf, 0, len); + Assert.assertEquals("Failed for value: " + value, value, decoded); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java new file mode 100644 index 0000000..6a7ed3b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java @@ -0,0 +1,166 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpZigZag; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Random; + +public class QwpZigZagTest { + + @Test + public void testEncodeDecodeInt() { + // Test 32-bit version + Assert.assertEquals(0, QwpZigZag.encode(0)); + Assert.assertEquals(0, QwpZigZag.decode(0)); + + Assert.assertEquals(2, QwpZigZag.encode(1)); + Assert.assertEquals(1, QwpZigZag.decode(2)); + + Assert.assertEquals(1, QwpZigZag.encode(-1)); + Assert.assertEquals(-1, QwpZigZag.decode(1)); + + int minInt = Integer.MIN_VALUE; + int encoded = QwpZigZag.encode(minInt); + Assert.assertEquals(minInt, QwpZigZag.decode(encoded)); + + int maxInt = Integer.MAX_VALUE; + encoded = QwpZigZag.encode(maxInt); + Assert.assertEquals(maxInt, QwpZigZag.decode(encoded)); + } + + @Test + public void testEncodeDecodeZero() { + Assert.assertEquals(0, QwpZigZag.encode(0L)); + Assert.assertEquals(0, QwpZigZag.decode(0L)); + } + + @Test + public void testEncodeMaxLong() { + long encoded = QwpZigZag.encode(Long.MAX_VALUE); + Assert.assertEquals(-2L, encoded); // 0xFFFFFFFFFFFFFFFE (all bits except LSB) + Assert.assertEquals(Long.MAX_VALUE, QwpZigZag.decode(encoded)); + } + + @Test + public void testEncodeMinLong() { + long encoded = QwpZigZag.encode(Long.MIN_VALUE); + Assert.assertEquals(-1L, encoded); // All bits set (unsigned max) + Assert.assertEquals(Long.MIN_VALUE, QwpZigZag.decode(encoded)); + } + + @Test + public void testEncodeNegative() { + // ZigZag encoding maps: + // -1 -> 1 + // -2 -> 3 + // -n -> 2n - 1 + Assert.assertEquals(1, QwpZigZag.encode(-1L)); + Assert.assertEquals(3, QwpZigZag.encode(-2L)); + Assert.assertEquals(5, QwpZigZag.encode(-3L)); + Assert.assertEquals(199, QwpZigZag.encode(-100L)); + } + + @Test + public void testEncodePositive() { + // ZigZag encoding maps: + // 0 -> 0 + // 1 -> 2 + // 2 -> 4 + // n -> 2n + Assert.assertEquals(2, QwpZigZag.encode(1L)); + Assert.assertEquals(4, QwpZigZag.encode(2L)); + Assert.assertEquals(6, QwpZigZag.encode(3L)); + Assert.assertEquals(200, QwpZigZag.encode(100L)); + } + + @Test + public void testEncodingPattern() { + // Verify the exact encoding pattern matches the formula: + // zigzag(n) = (n << 1) ^ (n >> 63) + // This means: + // - Non-negative n: zigzag(n) = 2 * n + // - Negative n: zigzag(n) = -2 * n - 1 + + for (int n = -100; n <= 100; n++) { + long encoded = QwpZigZag.encode((long) n); + long expected = (n >= 0) ? (2L * n) : (-2L * n - 1); + Assert.assertEquals("Encoding mismatch for n=" + n, expected, encoded); + } + } + + @Test + public void testRoundTripRandomValues() { + Random random = new Random(42); // Fixed seed for reproducibility + + for (int i = 0; i < 1000; i++) { + long value = random.nextLong(); + long encoded = QwpZigZag.encode(value); + long decoded = QwpZigZag.decode(encoded); + Assert.assertEquals("Failed for value: " + value, value, decoded); + } + } + + @Test + public void testSmallValuesHaveSmallEncodings() { + // The point of ZigZag is that small absolute values produce small encoded values + // which then encode efficiently as varints + + // -1 encodes to 1 (small, 1 byte as varint) + Assert.assertTrue(QwpZigZag.encode(-1L) < 128); + + // Small positive and negative values should encode to small values + // Values in [-63, 63] all encode to values < 128 (1 byte varint) + // 63 encodes to 126, -63 encodes to 125 + for (int n = -63; n <= 63; n++) { + long encoded = QwpZigZag.encode(n); + Assert.assertTrue("Value " + n + " encoded to " + encoded, + encoded < 128); // Fits in 1 byte varint + } + + // 64 encodes to 128, which requires 2 bytes as varint + Assert.assertEquals(128, QwpZigZag.encode(64L)); + } + + @Test + public void testSymmetry() { + // Test that encode then decode returns the original value + long[] testValues = { + 0, 1, -1, 2, -2, + 100, -100, + 1000000, -1000000, + Long.MAX_VALUE, Long.MIN_VALUE, + Long.MAX_VALUE / 2, Long.MIN_VALUE / 2 + }; + + for (long value : testValues) { + long encoded = QwpZigZag.encode(value); + long decoded = QwpZigZag.decode(encoded); + Assert.assertEquals("Failed for value: " + value, value, decoded); + } + } +} From 2ee3ac0ba4adbc186d163ce6f302e83616f11a2a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:15:33 +0100 Subject: [PATCH 079/230] Use VarHandle for InFlightWindow statistics totalAcked and totalFailed used volatile += (read-modify-write) without synchronization, which is racy when the acker thread updates concurrently with readers. Replace with VarHandle getAndAdd() for atomic updates and getOpaque() for tear-free reads, avoiding the overhead of AtomicLong while keeping the fields in-line with the rest of the lock-free design. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/InFlightWindow.java | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java index 869d242..6447cb8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java @@ -28,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; @@ -63,6 +65,18 @@ public class InFlightWindow { private static final long PARK_NANOS = 100_000; // 100 microseconds // Spin parameters private static final int SPIN_TRIES = 100; + private static final VarHandle TOTAL_ACKED; + private static final VarHandle TOTAL_FAILED; + + static { + try { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + TOTAL_ACKED = lookup.findVarHandle(InFlightWindow.class, "totalAcked", long.class); + TOTAL_FAILED = lookup.findVarHandle(InFlightWindow.class, "totalFailed", long.class); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } // Error state private final AtomicReference lastError = new AtomicReference<>(); private final int maxWindowSize; @@ -73,9 +87,9 @@ public class InFlightWindow { // Core state // highestSent: the sequence number of the last batch added to the window private volatile long highestSent = -1; - // Statistics (not strictly accurate under contention, but good enough for monitoring) - private volatile long totalAcked = 0; - private volatile long totalFailed = 0; + // Statistics — updated atomically via VarHandle + private long totalAcked = 0; + private long totalFailed = 0; // Thread waiting for empty (flush thread) private volatile Thread waitingForEmpty; // Thread waiting for space (sender thread) @@ -145,7 +159,7 @@ public int acknowledgeUpTo(long sequence) { highestAcked = effectiveSequence; int acknowledged = (int) (effectiveSequence - prevAcked); - totalAcked += acknowledged; + TOTAL_ACKED.getAndAdd(this, (long) acknowledged); LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount()); @@ -298,7 +312,7 @@ public void clearError() { public void fail(long batchId, Throwable error) { this.failedBatchId = batchId; this.lastError.set(error); - totalFailed++; + TOTAL_FAILED.getAndAdd(this, 1L); LOG.error("Batch failed [batchId={}, error={}]", batchId, String.valueOf(error)); @@ -320,7 +334,7 @@ public void failAll(Throwable error) { this.failedBatchId = sent; this.lastError.set(error); - totalFailed += Math.max(1, inFlight); + TOTAL_FAILED.getAndAdd(this, Math.max(1L, inFlight)); LOG.error("All in-flight batches failed [inFlight={}, error={}]", inFlight, String.valueOf(error)); @@ -356,14 +370,14 @@ public int getMaxWindowSize() { * Returns the total number of batches acknowledged. */ public long getTotalAcked() { - return totalAcked; + return (long) TOTAL_ACKED.getOpaque(this); } /** * Returns the total number of batches that failed. */ public long getTotalFailed() { - return totalFailed; + return (long) TOTAL_FAILED.getOpaque(this); } /** From 8a6cbe8db39195b2a8c1727a682b12244b6dbb7a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:26:32 +0100 Subject: [PATCH 080/230] Replace busy-wait with monitor wait in close() WebSocketSendQueue.close() busy-waited with Thread.sleep(10) outside the processingLock monitor to drain pending batches. This burned CPU unnecessarily and read pendingBuffer without holding the lock, risking a missed-update race. Replace the sleep loop with processingLock.wait(), matching the pattern already used by flush() and enqueue(). The I/O thread already calls processingLock.notifyAll() after polling a batch, so close() now wakes up promptly on notification or at the exact shutdown deadline. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/WebSocketSendQueue.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index f567d38..7b43fd9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -179,16 +179,19 @@ public void close() { // Wait for pending batches to be sent long startTime = System.currentTimeMillis(); - while (!isPendingEmpty()) { - if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { - LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); - break; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; + synchronized (processingLock) { + while (!isPendingEmpty()) { + long elapsed = System.currentTimeMillis() - startTime; + if (elapsed >= shutdownTimeoutMs) { + LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); + break; + } + try { + processingLock.wait(shutdownTimeoutMs - elapsed); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } } } From e1538067ac28b76fed01dd6596eb84e28f95f550 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:32:57 +0100 Subject: [PATCH 081/230] Validate Upgrade and Connection headers in WS handshake validateUpgradeResponse() previously only checked the HTTP 101 status line and Sec-WebSocket-Accept header. RFC 6455 Section 4.1 also requires the client to verify that the server response contains "Upgrade: websocket" and "Connection: Upgrade" headers. Add both checks with case-insensitive value matching as the RFC requires. The existing containsHeaderValue() helper gains an ignoreValueCase parameter so the Upgrade and Connection checks use equalsIgnoreCase while the Sec-WebSocket-Accept check retains its exact base64 match. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 9c6ca38..e015a04 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -449,7 +449,7 @@ public void upgrade(CharSequence path) { upgrade(path, defaultTimeout); } - private static boolean containsHeaderValue(String response, String headerName, String expectedValue) { + private static boolean containsHeaderValue(String response, String headerName, String expectedValue, boolean ignoreValueCase) { int headerLen = headerName.length(); int responseLen = response.length(); for (int i = 0; i <= responseLen - headerLen; i++) { @@ -460,7 +460,9 @@ private static boolean containsHeaderValue(String response, String headerName, S lineEnd = responseLen; } String actualValue = response.substring(valueStart, lineEnd).trim(); - return actualValue.equals(expectedValue); + return ignoreValueCase + ? actualValue.equalsIgnoreCase(expectedValue) + : actualValue.equals(expectedValue); } } return false; @@ -842,9 +844,19 @@ private void validateUpgradeResponse(int headerEnd) { throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine); } - // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230) + // Verify Upgrade: websocket (case-insensitive value per RFC 6455 Section 4.1) + if (!containsHeaderValue(response, "Upgrade:", "websocket", true)) { + throw new HttpClientException("Missing or invalid Upgrade header in WebSocket response"); + } + + // Verify Connection: Upgrade (case-insensitive value per RFC 6455 Section 4.1) + if (!containsHeaderValue(response, "Connection:", "Upgrade", true)) { + throw new HttpClientException("Missing or invalid Connection header in WebSocket response"); + } + + // Verify Sec-WebSocket-Accept (exact value match per RFC 6455 Section 4.1) String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey); - if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) { + if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept, false)) { throw new HttpClientException("Invalid Sec-WebSocket-Accept header"); } } From 859c7cb78b7eda9f57cc2395c2bf40c93530b0f7 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:38:55 +0100 Subject: [PATCH 082/230] Use SecureRnd for WebSocket handshake key Replace non-cryptographic Rnd (xorshift seeded with nanoTime/currentTimeMillis) with ChaCha20-based SecureRnd for generating the Sec-WebSocket-Key during the upgrade handshake. WebSocketSendBuffer already uses SecureRnd for frame masking, so this aligns the handshake key generation with the same standard. SecureRnd seeds once from SecureRandom at construction time, then produces unpredictable output with no heap allocations. The handshake key needs only 16 nextInt() calls (one ChaCha20 block) and runs once per connection, so the performance impact is negligible. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/http/client/WebSocketClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index e015a04..78a3245 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -37,7 +37,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Misc; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.Rnd; +import io.questdb.client.std.SecureRnd; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; import org.slf4j.Logger; @@ -76,7 +76,7 @@ public abstract class WebSocketClient implements QuietCloseable { private final int defaultTimeout; private final WebSocketFrameParser frameParser; private final int maxRecvBufSize; - private final Rnd rnd; + private final SecureRnd rnd; private final WebSocketSendBuffer sendBuffer; private boolean closed; private int fragmentBufPos; @@ -116,7 +116,7 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock this.recvReadPos = 0; this.frameParser = new WebSocketFrameParser(); - this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.rnd = new SecureRnd(); this.upgraded = false; this.closed = false; } @@ -405,7 +405,7 @@ public void upgrade(CharSequence path, int timeout) { // Generate random key byte[] keyBytes = new byte[16]; for (int i = 0; i < 16; i++) { - keyBytes[i] = (byte) rnd.nextInt(256); + keyBytes[i] = (byte) rnd.nextInt(); } handshakeKey = Base64.getEncoder().encodeToString(keyBytes); From 9e91498826e310d5a2bf01921f2b57e505ad34be Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:47:50 +0100 Subject: [PATCH 083/230] Clean up client QwpConstants Raise javac.target from 11 to 17 in the client module's pom.xml, matching the core module's language level. This enables enhanced switch expressions. Use enhanced switch expressions in getFixedTypeSize() and getTypeName(), matching the core module's style. Remove nine unused constants: CAPABILITY_REQUEST_SIZE, CAPABILITY_RESPONSE_SIZE, DEFAULT_MAX_STRING_LENGTH, HEADER_OFFSET_MAGIC, HEADER_OFFSET_PAYLOAD_LENGTH, HEADER_OFFSET_TABLE_COUNT, HEADER_OFFSET_VERSION, MAX_COLUMN_NAME_LENGTH, MAX_TABLE_NAME_LENGTH. Co-Authored-By: Claude Opus 4.6 --- core/pom.xml | 2 +- .../cutlass/qwp/protocol/QwpConstants.java | 173 ++++-------------- 2 files changed, 37 insertions(+), 138 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 1ccaa98..a6dfa9f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -32,7 +32,7 @@ false target none - 11 + 17 -ea -Dfile.encoding=UTF-8 -XX:+UseParallelGC None %regex[.*[^o].class] diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index a6142c9..a3ad097 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -29,14 +29,6 @@ */ public final class QwpConstants { - /** - * Size of capability request in bytes. - */ - public static final int CAPABILITY_REQUEST_SIZE = 8; - /** - * Size of capability response in bytes. - */ - public static final int CAPABILITY_RESPONSE_SIZE = 8; /** * Default initial receive buffer size (64 KB). */ @@ -54,10 +46,6 @@ public final class QwpConstants { * Default maximum rows per table in a batch. */ public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; - /** - * Default maximum string length in bytes (1 MB). - */ - public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; /** * Default maximum tables per batch. */ @@ -89,23 +77,6 @@ public final class QwpConstants { * Offset of flags byte in header. */ public static final int HEADER_OFFSET_FLAGS = 5; - /** - * Offset of magic bytes in header (4 bytes). - */ - public static final int HEADER_OFFSET_MAGIC = 0; - /** - * Offset of payload length (uint32, little-endian) in header. - */ - public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; - - /** - * Offset of table count (uint16, little-endian) in header. - */ - public static final int HEADER_OFFSET_TABLE_COUNT = 6; - /** - * Offset of version byte in header. - */ - public static final int HEADER_OFFSET_VERSION = 4; /** * Size of the message header in bytes. */ @@ -130,14 +101,6 @@ public final class QwpConstants { * Maximum columns per table (QuestDB limit). */ public static final int MAX_COLUMNS_PER_TABLE = 2048; - /** - * Maximum column name length in bytes. - */ - public static final int MAX_COLUMN_NAME_LENGTH = 127; - /** - * Maximum table name length in bytes. - */ - public static final int MAX_TABLE_NAME_LENGTH = 127; /** * Schema mode: Full schema included. */ @@ -307,35 +270,17 @@ private QwpConstants() { */ public static int getFixedTypeSize(byte typeCode) { int code = typeCode & TYPE_MASK; - switch (code) { - case TYPE_BOOLEAN: - return 0; // Special: bit-packed - case TYPE_BYTE: - return 1; - case TYPE_SHORT: - case TYPE_CHAR: - return 2; - case TYPE_INT: - case TYPE_FLOAT: - return 4; - case TYPE_LONG: - case TYPE_DOUBLE: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: - case TYPE_DECIMAL64: - return 8; - case TYPE_UUID: - case TYPE_DECIMAL128: - return 16; - case TYPE_LONG256: - case TYPE_DECIMAL256: - return 32; - case TYPE_GEOHASH: - return -1; // Variable width: varint precision + packed values - default: - return -1; // Variable width - } + return switch (code) { + case TYPE_BOOLEAN -> 0; // Special: bit-packed + case TYPE_BYTE -> 1; + case TYPE_SHORT, TYPE_CHAR -> 2; + case TYPE_INT, TYPE_FLOAT -> 4; + case TYPE_LONG, TYPE_DOUBLE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE, TYPE_DECIMAL64 -> 8; + case TYPE_UUID, TYPE_DECIMAL128 -> 16; + case TYPE_LONG256, TYPE_DECIMAL256 -> 32; + case TYPE_GEOHASH -> -1; // Variable width: varint precision + packed values + default -> -1; // Variable width + }; } /** @@ -347,77 +292,31 @@ public static int getFixedTypeSize(byte typeCode) { public static String getTypeName(byte typeCode) { int code = typeCode & TYPE_MASK; boolean nullable = (typeCode & TYPE_NULLABLE_FLAG) != 0; - String name; - switch (code) { - case TYPE_BOOLEAN: - name = "BOOLEAN"; - break; - case TYPE_BYTE: - name = "BYTE"; - break; - case TYPE_SHORT: - name = "SHORT"; - break; - case TYPE_CHAR: - name = "CHAR"; - break; - case TYPE_INT: - name = "INT"; - break; - case TYPE_LONG: - name = "LONG"; - break; - case TYPE_FLOAT: - name = "FLOAT"; - break; - case TYPE_DOUBLE: - name = "DOUBLE"; - break; - case TYPE_STRING: - name = "STRING"; - break; - case TYPE_SYMBOL: - name = "SYMBOL"; - break; - case TYPE_TIMESTAMP: - name = "TIMESTAMP"; - break; - case TYPE_TIMESTAMP_NANOS: - name = "TIMESTAMP_NANOS"; - break; - case TYPE_DATE: - name = "DATE"; - break; - case TYPE_UUID: - name = "UUID"; - break; - case TYPE_LONG256: - name = "LONG256"; - break; - case TYPE_GEOHASH: - name = "GEOHASH"; - break; - case TYPE_VARCHAR: - name = "VARCHAR"; - break; - case TYPE_DOUBLE_ARRAY: - name = "DOUBLE_ARRAY"; - break; - case TYPE_LONG_ARRAY: - name = "LONG_ARRAY"; - break; - case TYPE_DECIMAL64: - name = "DECIMAL64"; - break; - case TYPE_DECIMAL128: - name = "DECIMAL128"; - break; - case TYPE_DECIMAL256: - name = "DECIMAL256"; - break; - default: - name = "UNKNOWN(" + code + ")"; - } + String name = switch (code) { + case TYPE_BOOLEAN -> "BOOLEAN"; + case TYPE_BYTE -> "BYTE"; + case TYPE_SHORT -> "SHORT"; + case TYPE_CHAR -> "CHAR"; + case TYPE_INT -> "INT"; + case TYPE_LONG -> "LONG"; + case TYPE_FLOAT -> "FLOAT"; + case TYPE_DOUBLE -> "DOUBLE"; + case TYPE_STRING -> "STRING"; + case TYPE_SYMBOL -> "SYMBOL"; + case TYPE_TIMESTAMP -> "TIMESTAMP"; + case TYPE_TIMESTAMP_NANOS -> "TIMESTAMP_NANOS"; + case TYPE_DATE -> "DATE"; + case TYPE_UUID -> "UUID"; + case TYPE_LONG256 -> "LONG256"; + case TYPE_GEOHASH -> "GEOHASH"; + case TYPE_VARCHAR -> "VARCHAR"; + case TYPE_DOUBLE_ARRAY -> "DOUBLE_ARRAY"; + case TYPE_LONG_ARRAY -> "LONG_ARRAY"; + case TYPE_DECIMAL64 -> "DECIMAL64"; + case TYPE_DECIMAL128 -> "DECIMAL128"; + case TYPE_DECIMAL256 -> "DECIMAL256"; + default -> "UNKNOWN(" + code + ")"; + }; return nullable ? name + "?" : name; } From e6ef8e85df96ae902bb9c12a1d5860d9bc7124cf Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:15:21 +0100 Subject: [PATCH 084/230] Remove serverMode from client WebSocketFrameParser The client-side parser always operates in client mode (rejects masked frames, accepts unmasked). Remove the serverMode flag and setServerMode()/setStrictMode()/setMaskKey()/getMaskKey() methods that were only needed by the server-side copy. Add WebSocketFrameParserTest with 34 tests covering client-mode frame parsing: opcodes, length encodings, fragmentation, control frames, error cases, and masked-frame rejection. Co-Authored-By: Claude Opus 4.6 --- .../qwp/websocket/WebSocketFrameParser.java | 36 +- .../websocket/WebSocketFrameParserTest.java | 668 ++++++++++++++++++ 2 files changed, 671 insertions(+), 33 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index e9d2d54..85603d1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -73,8 +73,6 @@ public class WebSocketFrameParser { private boolean masked; private int opcode; private long payloadLength; - // Configuration - private boolean serverMode = false; // If true, expect masked frames from clients // Parser state private int state = STATE_HEADER; private boolean strictMode = false; // If true, reject non-minimal length encodings @@ -87,12 +85,6 @@ public int getHeaderSize() { return headerSize; } - public int getMaskKey() { - return maskKey; - } - - // Getters - public int getOpcode() { return opcode; } @@ -160,13 +152,9 @@ public int parse(long buf, long limit) { int lengthField = byte1 & LENGTH_MASK; // Validate masking based on mode - if (serverMode && !masked) { - // Client frames MUST be masked - state = STATE_ERROR; - errorCode = WebSocketCloseCode.PROTOCOL_ERROR; - return 0; - } - if (!serverMode && masked) { + // Configuration + // If true, expect masked frames from clients + if (masked) { // Server frames MUST NOT be masked state = STATE_ERROR; errorCode = WebSocketCloseCode.PROTOCOL_ERROR; @@ -274,24 +262,6 @@ public void reset() { errorCode = 0; } - /** - * Sets the mask key for unmasking. Used in testing. - */ - public void setMaskKey(int maskKey) { - this.maskKey = maskKey; - this.masked = true; - } - - // Setters for configuration - - public void setServerMode(boolean serverMode) { - this.serverMode = serverMode; - } - - public void setStrictMode(boolean strictMode) { - this.strictMode = strictMode; - } - /** * Unmasks the payload data in place. * diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java new file mode 100644 index 0000000..4bf7eb5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java @@ -0,0 +1,668 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.websocket; + +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +/** + * Tests for the client-side WebSocket frame parser. + * The client parser expects unmasked frames (from the server) + * and rejects masked frames. + */ +public class WebSocketFrameParserTest { + + @Test + public void testControlFrameBetweenFragments() { + long buf = allocateBuffer(64); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); + + // First data fragment + writeBytes(buf, (byte) 0x01, (byte) 0x02, (byte) 'H', (byte) 'i'); + parser.parse(buf, buf + 4); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + + // Ping in the middle (control frame, FIN must be 1) + parser.reset(); + writeBytes(buf, (byte) 0x89, (byte) 0x00); + parser.parse(buf, buf + 2); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + + // Final data fragment + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + parser.parse(buf, buf + 3); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + } + + @Test + public void testOpcodeIsControlFrame() { + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.TEXT)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.BINARY)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PONG)); + } + + @Test + public void testOpcodeIsDataFrame() { + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.CLOSE)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PING)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PONG)); + } + + @Test + public void testOpcodeIsValid() { + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isValid(3)); + Assert.assertFalse(WebSocketOpcode.isValid(4)); + Assert.assertFalse(WebSocketOpcode.isValid(5)); + Assert.assertFalse(WebSocketOpcode.isValid(6)); + Assert.assertFalse(WebSocketOpcode.isValid(7)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PONG)); + Assert.assertFalse(WebSocketOpcode.isValid(0xB)); + Assert.assertFalse(WebSocketOpcode.isValid(0xF)); + } + + @Test + public void testParse16BitLength() { + int payloadLen = 1000; + long buf = allocateBuffer(payloadLen + 16); + try { + writeBytes(buf, + (byte) 0x82, // FIN + BINARY + (byte) 126, // 16-bit length follows + (byte) (payloadLen >> 8), // Length high byte + (byte) (payloadLen & 0xFF) // Length low byte + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4 + payloadLen); + + Assert.assertEquals(4 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); + } finally { + freeBuffer(buf, payloadLen + 16); + } + } + + @Test + public void testParse64BitLength() { + long payloadLen = 70_000L; + long buf = allocateBuffer((int) payloadLen + 16); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x82); + Unsafe.getUnsafe().putByte(buf + 1, (byte) 127); + Unsafe.getUnsafe().putLong(buf + 2, Long.reverseBytes(payloadLen)); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 10 + payloadLen); + + Assert.assertEquals(10 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); + } finally { + freeBuffer(buf, (int) payloadLen + 16); + } + } + + @Test + public void testParse7BitLength() { + for (int len = 0; len <= 125; len++) { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x82, (byte) len); + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2 + len); + + Assert.assertEquals(2 + len, consumed); + Assert.assertEquals(len, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 256); + } + } + } + + @Test + public void testParseBinaryFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseCloseFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, + (byte) 0x88, // FIN + CLOSE + (byte) 0x02, // Length 2 (just the code) + (byte) 0x03, (byte) 0xE8 // 1000 in big-endian + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseCloseFrameEmpty() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseCloseFrameWithReason() { + long buf = allocateBuffer(64); + try { + String reason = "Normal closure"; + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + + Unsafe.getUnsafe().putByte(buf, (byte) 0x88); + Unsafe.getUnsafe().putByte(buf + 1, (byte) (2 + reasonBytes.length)); + Unsafe.getUnsafe().putShort(buf + 2, Short.reverseBytes((short) 1000)); + for (int i = 0; i < reasonBytes.length; i++) { + Unsafe.getUnsafe().putByte(buf + 4 + i, reasonBytes[i]); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4 + reasonBytes.length); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2 + reasonBytes.length, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 64); + } + } + + @Test + public void testParseContinuationFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x00, (byte) 0x05, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04, (byte) 0x05); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); + + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + Assert.assertFalse(parser.isFin()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseEmptyBuffer() { + long buf = allocateBuffer(16); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf); + + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseEmptyPayload() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2); + + Assert.assertEquals(2, consumed); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseFragmentedMessage() { + long buf = allocateBuffer(64); + try { + // First fragment: opcode=TEXT, FIN=0 + writeBytes(buf, (byte) 0x01, (byte) 0x03, (byte) 'H', (byte) 'e', (byte) 'l'); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 5); + + Assert.assertEquals(5, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + + // Continuation: opcode=CONTINUATION, FIN=0 + parser.reset(); + writeBytes(buf, (byte) 0x00, (byte) 0x02, (byte) 'l', (byte) 'o'); + consumed = parser.parse(buf, buf + 4); + + Assert.assertEquals(4, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + + // Final fragment: opcode=CONTINUATION, FIN=1 + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + } + + @Test + public void testParseIncompleteHeader16BitLength() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 126, (byte) 0x01); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseIncompleteHeader1Byte() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 1); + + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseIncompleteHeader64BitLength() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 127, (byte) 0, (byte) 0, (byte) 0, (byte) 0); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 6); + + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseIncompletePayload() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x05, (byte) 0x01, (byte) 0x02); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4); + + Assert.assertEquals(2, consumed); + Assert.assertEquals(5, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_PAYLOAD, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseMaxControlFrameSize() { + long buf = allocateBuffer(256); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x89); // PING + Unsafe.getUnsafe().putByte(buf + 1, (byte) 125); + for (int i = 0; i < 125; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 127); + + Assert.assertEquals(127, consumed); + Assert.assertEquals(125, parser.getPayloadLength()); + Assert.assertNotEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 256); + } + } + + @Test + public void testParseMinimalFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x01, (byte) 0xFF); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(1, parser.getPayloadLength()); + Assert.assertFalse(parser.isMasked()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseMultipleFramesInBuffer() { + long buf = allocateBuffer(32); + try { + writeBytes(buf, + (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02, + (byte) 0x81, (byte) 0x03, (byte) 'a', (byte) 'b', (byte) 'c' + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + + int consumed = parser.parse(buf, buf + 9); + Assert.assertEquals(4, consumed); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + consumed = parser.parse(buf + 4, buf + 9); + Assert.assertEquals(5, consumed); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + Assert.assertEquals(3, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 32); + } + } + + @Test + public void testParsePingFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x89, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); + + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + Assert.assertEquals(4, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParsePongFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x8A, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); + + Assert.assertEquals(WebSocketOpcode.PONG, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseTextFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x81, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectCloseFrameWith1BytePayload() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x01, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 3); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectFragmentedControlFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x09, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectMaskedFrame() { + // Client-side parser rejects masked frames from the server + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x81, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0xFF); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectOversizeControlFrame() { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x89, (byte) 126, (byte) 0x00, (byte) 0x7E); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 256); + } + } + + @Test + public void testRejectRSV2Bit() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0xA2, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectRSV3Bit() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x92, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectReservedBits() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0xC2, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectUnknownOpcode() { + for (int opcode : new int[]{3, 4, 5, 6, 7, 0xB, 0xC, 0xD, 0xE, 0xF}) { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) (0x80 | opcode), (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals("Opcode " + opcode + " should be rejected", + WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + } + + @Test + public void testReset() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + + Assert.assertEquals(0, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_HEADER, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + private static long allocateBuffer(int size) { + return Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + } + + private static void freeBuffer(long address, int size) { + Unsafe.free(address, size, MemoryTag.NATIVE_DEFAULT); + } + + private static void writeBytes(long address, byte... bytes) { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(address + i, bytes[i]); + } + } +} From 44f4db0e90b32e1478a4239cfe2607c0026795c9 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:16:01 +0100 Subject: [PATCH 085/230] Style cleanup in Sender --- .../main/java/io/questdb/client/Sender.java | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 705f3fe..33d2df2 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -145,20 +145,11 @@ static LineSenderBuilder builder(CharSequence configurationString) { * @return Builder object to create a new Sender instance. */ static LineSenderBuilder builder(Transport transport) { - int protocol; - switch (transport) { - case HTTP: - protocol = LineSenderBuilder.PROTOCOL_HTTP; - break; - case TCP: - protocol = LineSenderBuilder.PROTOCOL_TCP; - break; - case WEBSOCKET: - protocol = LineSenderBuilder.PROTOCOL_WEBSOCKET; - break; - default: - throw new LineSenderException("unknown transport: " + transport); - } + int protocol = switch (transport) { + case HTTP -> LineSenderBuilder.PROTOCOL_HTTP; + case TCP -> LineSenderBuilder.PROTOCOL_TCP; + case WEBSOCKET -> LineSenderBuilder.PROTOCOL_WEBSOCKET; + }; return new LineSenderBuilder(protocol); } @@ -562,7 +553,6 @@ final class LineSenderBuilder { private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; private String httpToken; - // WebSocket-specific fields private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY; private String keyId; private int maxBackoffMillis = PARAMETER_NOT_SET_EXPLICITLY; From 6867c50505e1fa4253814ed803cb75d68e307d50 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:19:27 +0100 Subject: [PATCH 086/230] Delete unused code --- .../cutlass/qwp/protocol/QwpBitReader.java | 137 ------------------ 1 file changed, 137 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index c1e2ec7..cb80a58 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -62,70 +62,6 @@ public class QwpBitReader { public QwpBitReader() { } - /** - * Aligns the reader to the next byte boundary by skipping any partial bits. - * - * @throws IllegalStateException if alignment fails - */ - public void alignToByte() { - int remainder = (int) (totalBitsRead % 8); - if (remainder != 0) { - skipBits(8 - remainder); - } - } - - /** - * Returns the number of bits remaining to be read. - * - * @return available bits - */ - public long getAvailableBits() { - return totalBitsAvailable - totalBitsRead; - } - - /** - * Returns the current position in bits from the start. - * - * @return bits read since reset - */ - public long getBitPosition() { - return totalBitsRead; - } - - /** - * Returns the current byte address being read. - * Note: This is approximate due to bit buffering. - * - * @return current address - */ - public long getCurrentAddress() { - return currentAddress; - } - - /** - * Returns true if there are more bits to read. - * - * @return true if bits available - */ - public boolean hasMoreBits() { - return totalBitsRead < totalBitsAvailable; - } - - /** - * Peeks at the next bit without consuming it. - * - * @return 0 or 1, or -1 if no more bits - */ - public int peekBit() { - if (totalBitsRead >= totalBitsAvailable) { - return -1; - } - if (!ensureBits(1)) { - return -1; - } - return (int) (bitBuffer & 1); - } - /** * Reads a single bit. * @@ -193,16 +129,6 @@ public long readBits(int numBits) { return result; } - /** - * Reads a complete byte, ensuring byte alignment first. - * - * @return the byte value (0-255) - * @throws IllegalStateException if not enough data - */ - public int readByte() { - return (int) readBits(8) & 0xFF; - } - /** * Reads a complete 32-bit integer in little-endian order. * @@ -213,16 +139,6 @@ public int readInt() { return (int) readBits(32); } - /** - * Reads a complete 64-bit long in little-endian order. - * - * @return the long value - * @throws IllegalStateException if not enough data - */ - public long readLong() { - return readBits(64); - } - /** * Reads multiple bits and interprets them as a signed value using two's complement. * @@ -255,59 +171,6 @@ public void reset(long address, long length) { this.totalBitsRead = 0; } - /** - * Resets the reader to read from the specified byte array. - * - * @param buf the byte array - * @param offset the starting offset - * @param length the number of bytes available - */ - public void reset(byte[] buf, int offset, int length) { - // For byte array, we'll store position info differently - // This is mainly for testing - in production we use direct memory - throw new UnsupportedOperationException("Use direct memory version"); - } - - /** - * Skips the specified number of bits. - * - * @param numBits bits to skip - * @throws IllegalStateException if not enough bits available - */ - public void skipBits(int numBits) { - if (totalBitsRead + numBits > totalBitsAvailable) { - throw new IllegalStateException("bit read overflow"); - } - - // Fast path: skip bits in current buffer - if (numBits <= bitsInBuffer) { - bitBuffer >>>= numBits; - bitsInBuffer -= numBits; - totalBitsRead += numBits; - return; - } - - // Consume all buffered bits - int bitsToSkip = numBits - bitsInBuffer; - totalBitsRead += bitsInBuffer; - bitsInBuffer = 0; - bitBuffer = 0; - - // Skip whole bytes - int bytesToSkip = bitsToSkip / 8; - currentAddress += bytesToSkip; - totalBitsRead += bytesToSkip * 8L; - - // Handle remaining bits - int remainingBits = bitsToSkip % 8; - if (remainingBits > 0) { - ensureBits(remainingBits); - bitBuffer >>>= remainingBits; - bitsInBuffer -= remainingBits; - totalBitsRead += remainingBits; - } - } - /** * Ensures the buffer has at least the requested number of bits. * Loads more bytes from memory if needed. From 099c183184462730d5ab274b3fb5adbc45cedefd Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:21:50 +0100 Subject: [PATCH 087/230] Clean up WebSocketClientFactory --- .../http/client/WebSocketClientFactory.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java index 9284786..c6b36d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java @@ -63,6 +63,10 @@ */ public class WebSocketClientFactory { + // Utility class -- no instantiation + private WebSocketClientFactory() { + } + /** * Creates a new WebSocket client with insecure TLS (no certificate validation). *

    @@ -82,17 +86,12 @@ public static WebSocketClient newInsecureTlsInstance() { * @return a new platform-specific WebSocket client */ public static WebSocketClient newInstance(HttpClientConfiguration configuration, SocketFactory socketFactory) { - switch (Os.type) { - case Os.LINUX: - return new WebSocketClientLinux(configuration, socketFactory); - case Os.DARWIN: - case Os.FREEBSD: - return new WebSocketClientOsx(configuration, socketFactory); - case Os.WINDOWS: - return new WebSocketClientWindows(configuration, socketFactory); - default: - throw new UnsupportedOperationException("Unsupported platform: " + Os.type); - } + return switch (Os.type) { + case Os.LINUX -> new WebSocketClientLinux(configuration, socketFactory); + case Os.DARWIN, Os.FREEBSD -> new WebSocketClientOsx(configuration, socketFactory); + case Os.WINDOWS -> new WebSocketClientWindows(configuration, socketFactory); + default -> throw new UnsupportedOperationException("Unsupported platform: " + Os.type); + }; } /** From 254c7c3550dc41995572433fe63715a54d82f29a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:25:34 +0100 Subject: [PATCH 088/230] Fix copyright year --- .../cutlass/qwp/protocol/QwpBitReader.java | 2 +- .../cutlass/qwp/protocol/QwpBitWriter.java | 2 +- .../cutlass/qwp/protocol/QwpColumnDef.java | 2 +- .../cutlass/qwp/protocol/QwpConstants.java | 2 +- .../cutlass/qwp/protocol/QwpNullBitmap.java | 2 +- .../cutlass/qwp/protocol/QwpSchemaHash.java | 2 +- .../cutlass/qwp/protocol/QwpVarint.java | 2 +- .../cutlass/qwp/protocol/QwpZigZag.java | 2 +- .../qwp/websocket/WebSocketFrameWriter.java | 131 ------------------ .../line/tcp/v4/QwpAllocationTestClient.java | 2 +- .../line/tcp/v4/StacBenchmarkClient.java | 2 +- 11 files changed, 10 insertions(+), 141 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index cb80a58..a253f6e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index 3f18d2d..30173cb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index f59a009..8c2c65a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index a3ad097..8c36d93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java index dd6020e..f78f65e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index b62fc07..94b3693 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java index ad675f4..f02a4ca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java index f113460..512de7d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index 892fed4..2f3c653 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -26,8 +26,6 @@ import io.questdb.client.std.Unsafe; -import java.nio.charset.StandardCharsets; - /** * Zero-allocation WebSocket frame writer. * Writes WebSocket frames according to RFC 6455. @@ -99,78 +97,6 @@ public static void maskPayload(long buf, long len, int maskKey) { } } - /** - * Writes a binary frame with payload from a memory address. - * - * @param buf the buffer to write to - * @param payloadPtr pointer to the payload data - * @param payloadLen length of payload - * @return the total number of bytes written - */ - public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - - // Copy payload from memory - Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); - - return headerLen + payloadLen; - } - - /** - * Writes a binary frame header only (for when payload is written separately). - * - * @param buf the buffer to write to - * @param payloadLen length of payload that will follow - * @return the header size in bytes - */ - public static int writeBinaryFrameHeader(long buf, int payloadLen) { - return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - } - - /** - * Writes a complete Close frame to the buffer. - * - * @param buf the buffer to write to - * @param code the close status code - * @param reason the close reason (may be null) - * @return the total number of bytes written (header + payload) - */ - public static int writeCloseFrame(long buf, int code, String reason) { - int payloadLen = 2; // status code - if (reason != null && !reason.isEmpty()) { - payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; - } - - int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); - int payloadOffset = writeClosePayload(buf + headerLen, code, reason); - - return headerLen + payloadOffset; - } - - /** - * Writes the payload for a Close frame. - * - * @param buf the buffer to write to (after the header) - * @param code the close status code - * @param reason the close reason (may be null) - * @return the number of bytes written - */ - public static int writeClosePayload(long buf, int code, String reason) { - // Write status code in network byte order (big-endian) - Unsafe.getUnsafe().putShort(buf, Short.reverseBytes((short) code)); - int offset = 2; - - // Write reason if provided - if (reason != null && !reason.isEmpty()) { - byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); - for (byte reasonByte : reasonBytes) { - Unsafe.getUnsafe().putByte(buf + offset++, reasonByte); - } - } - - return offset; - } - /** * Writes a WebSocket frame header to the buffer. * @@ -221,61 +147,4 @@ public static int writeHeader(long buf, boolean fin, int opcode, long payloadLen Unsafe.getUnsafe().putInt(buf + offset, maskKey); return offset + 4; } - - /** - * Writes a complete Ping frame to the buffer. - * - * @param buf the buffer to write to - * @param payload the ping payload - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write - * @return the total number of bytes written - */ - public static int writePingFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.PING, payloadLen, false); - - // Copy payload - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); - } - - return headerLen + payloadLen; - } - - /** - * Writes a complete Pong frame to the buffer. - * - * @param buf the buffer to write to - * @param payload the pong payload (should match the received ping) - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write - * @return the total number of bytes written - */ - public static int writePongFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); - - // Copy payload - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); - } - - return headerLen + payloadLen; - } - - /** - * Writes a Pong frame with payload from a memory address. - * - * @param buf the buffer to write to - * @param payloadPtr pointer to the ping payload to echo - * @param payloadLen length of payload - * @return the total number of bytes written - */ - public static int writePongFrame(long buf, long payloadPtr, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); - - // Copy payload from memory - Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); - - return headerLen + payloadLen; - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 21f47b0..46c0a9c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index 289643c..1eb044d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 2a14356060df4fc0ca57ba171e6115377d358f57 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:30:34 +0100 Subject: [PATCH 089/230] Delete duplicate method utf8Length() --- .../http/client/WebSocketSendBuffer.java | 3 ++- .../qwp/client/NativeBufferWriter.java | 3 +++ .../cutlass/qwp/client/QwpBufferWriter.java | 27 ------------------- .../qwp/client/NativeBufferWriterTest.java | 9 +++---- 4 files changed, 9 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index afb0d05..d31044f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.http.client; import io.questdb.client.cutlass.line.array.ArrayBufferAppender; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; @@ -336,7 +337,7 @@ public void putString(String value) { putVarint(0); return; } - int utf8Len = QwpBufferWriter.utf8Length(value); + int utf8Len = NativeBufferWriter.utf8Length(value); putVarint(utf8Len); putUtf8(value); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 5d8eb9e..e538769 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -56,6 +56,9 @@ public NativeBufferWriter(int initialCapacity) { /** * Returns the UTF-8 encoded length of a string. + * + * @param s the string (may be null) + * @return the number of bytes needed to encode the string as UTF-8 */ public static int utf8Length(String s) { if (s == null) return 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java index f32f575..644fdf8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java @@ -44,33 +44,6 @@ */ public interface QwpBufferWriter extends ArrayBufferAppender { - /** - * Returns the UTF-8 encoded length of a string. - * - * @param s the string (may be null) - * @return the number of bytes needed to encode the string as UTF-8 - */ - static int utf8Length(String s) { - if (s == null) return 0; - int len = 0; - for (int i = 0, n = s.length(); i < n; i++) { - char c = s.charAt(i); - if (c < 0x80) { - len++; - } else if (c < 0x800) { - len += 2; - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { - i++; - len += 4; - } else if (Character.isSurrogate(c)) { - len++; - } else { - len += 3; - } - } - return len; - } - /** * Ensures the buffer has capacity for at least the specified * additional bytes beyond the current position. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index c77f8d3..5bf342d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -25,7 +25,6 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; -import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; @@ -206,13 +205,13 @@ public void testPutUtf8LoneSurrogateMatchesUtf8Length() { @Test public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 - assertEquals(2, QwpBufferWriter.utf8Length("\uD800X")); + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); // Lone high surrogate at end: '?' (1) - assertEquals(1, QwpBufferWriter.utf8Length("\uD800")); + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); // Lone low surrogate: '?' (1) - assertEquals(1, QwpBufferWriter.utf8Length("\uDC00")); + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); // Valid pair still works: 4 bytes - assertEquals(4, QwpBufferWriter.utf8Length("\uD83D\uDE00")); + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); } @Test From a8fa83619f1f74ad9c1db209eb90b82622c8a944 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 08:39:22 +0100 Subject: [PATCH 090/230] Add async mode integration tests Add AsyncModeIntegrationTest with 6 tests that exercise double-buffering, send queue, and in-flight window working together. The tests use FakeWebSocketClient to simulate server behavior without requiring a running QuestDB instance. Tests cover: - Buffer cycling through all states across 2 alternating buffers (FILLING -> SEALED -> SENDING -> RECYCLED) - Backpressure when the in-flight window is full, blocking enqueue until ACKs arrive - Buffer swap waiting for a slow send to complete before the buffer can be reused - flush() returning after send but before ACK, requiring a separate awaitEmpty() call - High throughput with 50 batches and one-at-a-time ACKs - Server error propagation through the pipeline Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/AsyncModeIntegrationTest.java | 601 ++++++++++++++++++ 1 file changed, 601 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java new file mode 100644 index 0000000..edac6e9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java @@ -0,0 +1,601 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +/** + * Integration tests for async mode: double-buffering, send queue, and + * in-flight window working together. + *

    + * These tests verify the interaction between the three async mode components + * ({@link MicrobatchBuffer}, {@link WebSocketSendQueue}, {@link InFlightWindow}) + * without requiring a running QuestDB server. They use {@link FakeWebSocketClient} + * to simulate server behavior and control ACK timing. + */ +public class AsyncModeIntegrationTest { + + /** + * Window of 2. Sends 2 batches (fills window), then enqueues a 3rd to + * occupy the pending slot. The 4th enqueue blocks because the pending + * slot is occupied and the I/O thread cannot poll it (window full). + * Delivering ACKs unblocks the pipeline. + */ + @Test + public void testBackpressureBlocksEnqueueUntilAck() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + CountDownLatch twoSent = new CountDownLatch(2); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + + client.setSendBehavior((ptr, len) -> { + highestSent.incrementAndGet(); + twoSent.countDown(); + }); + client.setTryReceiveBehavior(handler -> { + if (deliverAcks.get()) { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 3_000, 500); + + // Send 2 batches to fill the window. + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); + + assertTrue("Both batches should be sent", twoSent.await(2, TimeUnit.SECONDS)); + assertEquals("Window should be full", 2, window.getInFlightCount()); + + // Reuse buf0 (recycled by I/O thread) and enqueue a 3rd batch. + // The I/O thread cannot poll it because the window is full. + assertTrue(buf0.awaitRecycled(2, TimeUnit.SECONDS)); + buf0.reset(); + buf0.writeByte((byte) 3); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + // Reuse buf1 and try to enqueue a 4th batch on a background + // thread. It should block because the pending slot is still + // occupied by the 3rd batch. + assertTrue(buf1.awaitRecycled(2, TimeUnit.SECONDS)); + buf1.reset(); + buf1.writeByte((byte) 4); + buf1.incrementRowCount(); + buf1.seal(); + + CountDownLatch enqueueStarted = new CountDownLatch(1); + CountDownLatch enqueueDone = new CountDownLatch(1); + AtomicReference errorRef = new AtomicReference<>(); + WebSocketSendQueue q = queue; + + Thread enqueueThread = new Thread(() -> { + enqueueStarted.countDown(); + try { + q.enqueue(buf1); + } catch (Throwable t) { + errorRef.set(t); + } finally { + enqueueDone.countDown(); + } + }); + enqueueThread.start(); + + assertTrue(enqueueStarted.await(1, TimeUnit.SECONDS)); + Thread.sleep(200); + assertEquals("Enqueue should still be blocked", 1, enqueueDone.getCount()); + + // Deliver ACKs to unblock the pipeline. + deliverAcks.set(true); + + assertTrue("Enqueue should complete after ACK", enqueueDone.await(3, TimeUnit.SECONDS)); + assertNull("No error expected", errorRef.get()); + + queue.flush(); + window.awaitEmpty(); + } finally { + deliverAcks.set(true); + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + } + + /** + * Sends 10 batches through 2 alternating buffers with auto-ACK. + * Each buffer cycles through all states multiple times: + * FILLING -> SEALED -> SENDING -> RECYCLED -> FILLING. + */ + @Test + public void testBatchesCycleThroughDoubleBuffers() throws Exception { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + int batchCount = 10; + + try { + queue = new WebSocketSendQueue(client, window, 5_000, 500); + MicrobatchBuffer active = buf0; + + for (int i = 0; i < batchCount; i++) { + if (active.isRecycled()) { + active.reset(); + } + assertTrue("Buffer should be FILLING on iteration " + i, active.isFilling()); + + active.writeByte((byte) (i & 0xFF)); + active.incrementRowCount(); + active.seal(); + queue.enqueue(active); + + // Swap to the other buffer, waiting for it if still in use. + MicrobatchBuffer other = (active == buf0) ? buf1 : buf0; + if (other.isInUse()) { + assertTrue("Other buffer should recycle", + other.awaitRecycled(2, TimeUnit.SECONDS)); + } + if (other.isRecycled()) { + other.reset(); + } + active = other; + } + + queue.flush(); + window.awaitEmpty(); + + assertEquals(batchCount, queue.getTotalBatchesSent()); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + } + + /** + * The first send blocks in sendBinary (simulating slow I/O). + * The user enqueues a second batch, then tries to swap back to the + * first buffer which is still in SENDING state. The user must wait + * until the I/O thread finishes and recycles the buffer. + */ + @Test + public void testBufferSwapWaitsForSlowSend() throws Exception { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + CountDownLatch sendStarted = new CountDownLatch(1); + CountDownLatch sendGate = new CountDownLatch(1); + + client.setSendBehavior((ptr, len) -> { + long seq = highestSent.incrementAndGet(); + if (seq == 0) { + // Block on first send to simulate slow I/O. + sendStarted.countDown(); + try { + if (!sendGate.await(5, TimeUnit.SECONDS)) { + throw new RuntimeException("sendGate timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 5_000, 500); + + // Enqueue buf0. The I/O thread starts sending and blocks. + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + assertTrue("I/O thread should start sending", sendStarted.await(2, TimeUnit.SECONDS)); + assertTrue("buf0 should be in use (SENDING)", buf0.isInUse()); + + // Enqueue buf1 into the pending slot (I/O thread is blocked). + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); + + // The user wants to reuse buf0, but it is still SENDING. + assertTrue("buf0 should still be in use", buf0.isInUse()); + + // Release the gate so the I/O thread can finish sending buf0. + sendGate.countDown(); + + // buf0 transitions SENDING -> RECYCLED. + assertTrue("buf0 should be recycled after send completes", + buf0.awaitRecycled(2, TimeUnit.SECONDS)); + assertTrue(buf0.isRecycled()); + + // Reset and verify buf0 is reusable. + buf0.reset(); + assertTrue(buf0.isFilling()); + + queue.flush(); + window.awaitEmpty(); + assertEquals(2, queue.getTotalBatchesSent()); + } finally { + sendGate.countDown(); + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + } + + /** + * Verifies that {@link WebSocketSendQueue#flush()} returns once the + * batch has been sent over the wire, even though the server has not + * ACKed it yet. The caller must separately call + * {@link InFlightWindow#awaitEmpty()} to wait for the ACK. + */ + @Test + public void testFlushWaitsForSendButNotForAcks() throws Exception { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + if (deliverAcks.get()) { + long sent = highestSent.get(); + if (sent >= 0 && window.getInFlightCount() > 0) { + emitAck(handler, sent); + return true; + } + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 2_000, 500); + + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + // flush() returns once the batch is sent, not when ACKed. + queue.flush(); + assertEquals(1, queue.getTotalBatchesSent()); + assertEquals("Batch should still be in flight", 1, window.getInFlightCount()); + + // Deliver ACK and wait for the window to drain. + deliverAcks.set(true); + window.awaitEmpty(); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + client.close(); + } + } + + /** + * Sends 50 batches through 2 buffers with a window of 4. + * ACKs arrive one-at-a-time (non-cumulative) to test sustained flow + * control under moderate backpressure. + */ + @Test + public void testHighThroughputWithManyBatches() throws Exception { + int batchCount = 50; + int windowSize = 4; + + InFlightWindow window = new InFlightWindow(windowSize, 10_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + // ACK one batch at a time to test sustained flow. + long next = acked + 1; + highestAcked.set(next); + emitAck(handler, next); + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 10_000, 2_000); + MicrobatchBuffer active = buf0; + + for (int i = 0; i < batchCount; i++) { + if (!active.isFilling()) { + if (active.isInUse()) { + assertTrue("Buffer should recycle on iteration " + i, + active.awaitRecycled(5, TimeUnit.SECONDS)); + } + active.reset(); + } + + active.writeByte((byte) (i & 0xFF)); + active.incrementRowCount(); + active.seal(); + queue.enqueue(active); + + active = (active == buf0) ? buf1 : buf0; + } + + queue.flush(); + window.awaitEmpty(); + + assertEquals(batchCount, queue.getTotalBatchesSent()); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + } + + /** + * The server ACKs the first batch but returns a WRITE_ERROR for the + * second. {@link WebSocketSendQueue#flush()} completes (both batches + * were sent) but {@link InFlightWindow#awaitEmpty()} surfaces the error. + */ + @Test + public void testServerErrorPropagatesOnFlush() throws Exception { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestDelivered = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long delivered = highestDelivered.get(); + if (sent > delivered) { + long next = delivered + 1; + highestDelivered.set(next); + if (next == 1) { + emitError(handler, next, WebSocketResponse.STATUS_WRITE_ERROR, "disk full"); + } else { + emitAck(handler, next); + } + return true; + } + return false; + }); + + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + + try { + queue = new WebSocketSendQueue(client, window, 2_000, 500); + + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); + + // flush() waits for the queue to drain (both batches sent). + queue.flush(); + + // awaitEmpty() surfaces the server error for batch 1. + try { + window.awaitEmpty(); + fail("Expected server error to propagate"); + } catch (LineSenderException e) { + assertTrue("Error should mention server failure", + e.getMessage().contains("disk full") || e.getMessage().contains("Server error")); + } + } finally { + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + } + + private static void closeQuietly(WebSocketSendQueue queue) { + if (queue != null) { + queue.close(); + } + } + + private static void emitAck(WebSocketFrameHandler handler, long sequence) { + WebSocketResponse resp = WebSocketResponse.success(sequence); + int size = resp.serializedSize(); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + resp.writeTo(ptr); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void emitError(WebSocketFrameHandler handler, long sequence, byte status, String message) { + WebSocketResponse resp = WebSocketResponse.error(sequence, status, message); + int size = resp.serializedSize(); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + resp.writeTo(ptr); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private interface SendBehavior { + void send(long dataPtr, int length); + } + + private interface TryReceiveBehavior { + boolean tryReceive(WebSocketFrameHandler handler); + } + + private static class FakeWebSocketClient extends WebSocketClient { + private volatile boolean connected = true; + private volatile SendBehavior sendBehavior = (dataPtr, length) -> {}; + private volatile TryReceiveBehavior tryReceiveBehavior = handler -> false; + + private FakeWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void close() { + connected = false; + super.close(); + } + + @Override + public boolean isConnected() { + return connected; + } + + @Override + public void sendBinary(long dataPtr, int length) { + sendBehavior.send(dataPtr, length); + } + + public void setSendBehavior(SendBehavior sendBehavior) { + this.sendBehavior = sendBehavior; + } + + public void setTryReceiveBehavior(TryReceiveBehavior tryReceiveBehavior) { + this.tryReceiveBehavior = tryReceiveBehavior; + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + return tryReceiveBehavior.tryReceive(handler); + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} From 266abfb8a67c3ffa0ec4f412d6fbb285d75181c1 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 09:33:08 +0100 Subject: [PATCH 091/230] Add cancelRow unit tests to QwpWebSocketSenderTest Add two unit tests for cancelRow() on QwpWebSocketSender: - testCancelRowDiscardsPartialRow: verifies that calling cancelRow() after table()/longColumn()/boolColumn() discards the in-progress row, leaving zero committed rows in the buffer. - testCancelRowNoOpWithoutTable: verifies that calling cancelRow() without a prior table() call is a safe no-op. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSenderTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index a128ed3..6ce9885 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -30,6 +30,7 @@ import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.network.PlainSocketFactory; import org.junit.Assert; import org.junit.Test; @@ -121,6 +122,30 @@ public void testCancelRowAfterCloseThrows() { } } + @Test + public void testCancelRowDiscardsPartialRow() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.table("test"); + sender.longColumn("x", 1); + sender.boolColumn("y", true); + + // Row is not yet committed (no at/atNow call), cancel it + sender.cancelRow(); + + // Buffer should have no committed rows + QwpTableBuffer buf = sender.getTableBuffer("test"); + Assert.assertEquals(0, buf.getRowCount()); + } + } + + @Test + public void testCancelRowNoOpWithoutTable() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + // cancelRow without table() should be a no-op (no NPE) + sender.cancelRow(); + } + } + @Test public void testCloseIdemponent() { QwpWebSocketSender sender = createUnconnectedSender(); From 3ab006bbaa8038150bede573ca7ab2f8a10e10af Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 10:49:30 +0100 Subject: [PATCH 092/230] Add assertMemoryLeak to client tests Wrap native-memory-allocating tests in assertMemoryLeak() across 16 test files in java-questdb-client. This ensures leak detection catches any native memory that isn't properly freed. Fix real resource leaks found during the audit: - QwpWebSocketEncoderTest: close ~8 QwpTableBuffer instances that were created without try-with-resources - QwpWebSocketSenderTest: close sender in 3 testTableBefore* tests that never called close() - DeltaSymbolDictionaryTest: close ~8 QwpTableBuffer instances that were created without try-with-resources Convert all existing TestUtils.assertMemoryLeak() calls to use a static import for consistency. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/HttpHeaderParserTest.java | 7 +- .../http/client/WebSocketClientTest.java | 6 +- .../http/client/WebSocketSendBufferTest.java | 21 +- .../test/cutlass/json/JsonLexerTest.java | 5 +- .../cutlass/line/LineSenderBuilderTest.java | 1 + .../line/tcp/PlainTcpLineChannelTest.java | 12 +- .../cutlass/line/udp/UdpLineChannelTest.java | 8 +- .../qwp/client/AsyncModeIntegrationTest.java | 725 ++++--- .../qwp/client/DeltaSymbolDictionaryTest.java | 849 ++++---- .../LineSenderBuilderWebSocketTest.java | 85 +- .../qwp/client/MicrobatchBufferTest.java | 82 +- .../qwp/client/NativeBufferWriterTest.java | 715 +++--- .../qwp/client/QwpDeltaDictRollbackTest.java | 4 +- .../qwp/client/QwpWebSocketEncoderTest.java | 1910 +++++++++-------- .../client/QwpWebSocketSenderStateTest.java | 8 +- .../qwp/client/QwpWebSocketSenderTest.java | 522 +++-- .../qwp/client/WebSocketSendQueueTest.java | 353 +-- .../qwp/protocol/OffHeapAppendMemoryTest.java | 541 ++--- .../qwp/protocol/QwpBitWriterTest.java | 245 ++- .../qwp/protocol/QwpGorillaEncoderTest.java | 1009 +++++---- .../qwp/protocol/QwpNullBitmapTest.java | 453 ++-- .../qwp/protocol/QwpSchemaHashTest.java | 31 +- .../qwp/protocol/QwpTableBufferTest.java | 717 ++++--- .../cutlass/qwp/protocol/QwpVarintTest.java | 91 +- .../websocket/WebSocketFrameParserTest.java | 947 ++++---- .../questdb/client/test/std/VectFuzzTest.java | 4 +- 26 files changed, 4969 insertions(+), 4382 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java index f7a9201..4cfc8c0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/HttpHeaderParserTest.java @@ -33,6 +33,7 @@ import io.questdb.client.std.str.DirectUtf8String; import io.questdb.client.std.str.Utf8String; import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -53,7 +54,7 @@ public class HttpHeaderParserTest { @Test public void testContentLengthLarge() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String v = "Content-Length: 81136060058\r\n" + "\r\n"; long p = TestUtils.toMemory(v); @@ -135,7 +136,7 @@ public void testProtocolLineFuzz() { @Test public void testQueryDanglingEncoding() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String v = "GET /status?x=1&a=% HTTP/1.1\r\n" + "\r\n"; long p = TestUtils.toMemory(v); @@ -152,7 +153,7 @@ public void testQueryDanglingEncoding() throws Exception { @Test public void testQueryInvalidEncoding() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String v = "GET /status?x=1&a=%i6b&c&d=x HTTP/1.1\r\n" + "\r\n"; long p = TestUtils.toMemory(v); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java index 34dd0bd..647c722 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -30,7 +30,7 @@ import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.Unsafe; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -40,7 +40,7 @@ public class WebSocketClientTest { @Test public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (StubWebSocketClient client = new StubWebSocketClient()) { WebSocketSendBuffer sendBuffer = client.getSendBuffer(); @@ -69,7 +69,7 @@ public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception { @Test public void testSendPingDoesNotClobberSendBuffer() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (StubWebSocketClient client = new StubWebSocketClient()) { // Set upgraded=true so checkConnected() passes setField(client, "upgraded", true); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java index 2218e9d..606c729 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -33,14 +34,16 @@ public class WebSocketSendBufferTest { @Test - public void testPutUtf8InvalidSurrogatePair() { - try (WebSocketSendBuffer buf = new WebSocketSendBuffer(256)) { - // High surrogate \uD800 followed by non-low-surrogate 'X'. - // Should produce '?' for the lone high surrogate, then 'X'. - buf.putUtf8("\uD800X"); - assertEquals(2, buf.getWritePos()); - assertEquals((byte) '?', Unsafe.getUnsafe().getByte(buf.getBufferPtr())); - assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 1)); - } + public void testPutUtf8InvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + try (WebSocketSendBuffer buf = new WebSocketSendBuffer(256)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + buf.putUtf8("\uD800X"); + assertEquals(2, buf.getWritePos()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(buf.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 1)); + } + }); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java index fafad57..9f7f067 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/json/JsonLexerTest.java @@ -33,6 +33,7 @@ import io.questdb.client.std.Mutable; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -78,7 +79,7 @@ public void testBreakOnValue() throws Exception { @Test public void testCacheDisabled() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String json = "{\"a\":1, \"b\": \"123456789012345678901234567890\"}"; int len = json.length(); long address = TestUtils.toMemory(json); @@ -251,7 +252,7 @@ public void testSimpleJson() throws Exception { @Test public void testStringTooLong() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { String json = "{\"a\":1, \"b\": \"123456789012345678901234567890\"]}"; int len = json.length() - 6; long address = TestUtils.toMemory(json); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java index 0fcad48..031f204 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -27,6 +27,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java index 712cf72..f95f8ab 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/PlainTcpLineChannelTest.java @@ -28,7 +28,7 @@ import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.fail; @@ -44,7 +44,7 @@ public int socketTcp(boolean blocking) { @Test public void testConstructorLeak_Hostname_CannotConnect() throws Exception { NetworkFacade nf = NetworkFacadeImpl.INSTANCE; - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(nf, "localhost", 1000, 1000); fail("there should be nothing listening on the port 1000, the channel should have failed to connect"); @@ -57,7 +57,7 @@ public void testConstructorLeak_Hostname_CannotConnect() throws Exception { @Test public void testConstructorLeak_Hostname_CannotResolveHost() throws Exception { NetworkFacade nf = NetworkFacadeImpl.INSTANCE; - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(nf, "nonsense-fails-to-resolve", 1000, 1000); fail("the host should not resolved and the channel should have failed to connect"); @@ -69,7 +69,7 @@ public void testConstructorLeak_Hostname_CannotResolveHost() throws Exception { @Test public void testConstructorLeak_Hostname_DescriptorsExhausted() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(FD_EXHAUSTED_NET_FACADE, "localhost", 1000, 1000); fail("the channel should fail to instantiate when NF fails to create a new socket"); @@ -82,7 +82,7 @@ public void testConstructorLeak_Hostname_DescriptorsExhausted() throws Exception @Test public void testConstructorLeak_IP_CannotConnect() throws Exception { NetworkFacade nf = NetworkFacadeImpl.INSTANCE; - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(nf, -1, 1000, 1000); fail("the channel should have failed to connect to address -1"); @@ -94,7 +94,7 @@ public void testConstructorLeak_IP_CannotConnect() throws Exception { @Test public void testConstructorLeak_IP_DescriptorsExhausted() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new PlainTcpLineChannel(FD_EXHAUSTED_NET_FACADE, -1, 1000, 1000); fail("the channel should fail to instantiate when NF fails to create a new socket"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java index 95fed98..cf3619f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/UdpLineChannelTest.java @@ -28,7 +28,7 @@ import io.questdb.client.cutlass.line.udp.UdpLineChannel; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.fail; @@ -55,7 +55,7 @@ public int socketUdp() { @Test public void testConstructorLeak_DescriptorsExhausted() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new UdpLineChannel(FD_EXHAUSTED_NET_FACADE, 1, 1, 9000, 10); fail("the channel should fail to instantiate when NetworkFacade fails to create a new socket"); @@ -67,7 +67,7 @@ public void testConstructorLeak_DescriptorsExhausted() throws Exception { @Test public void testConstructorLeak_FailsToSendInterface() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new UdpLineChannel(FAILS_TO_SET_MULTICAST_IFACE_NET_FACADE, 1, 1, 9000, 10); fail("the channel should fail to instantiate when NF fails to set multicast interface"); @@ -79,7 +79,7 @@ public void testConstructorLeak_FailsToSendInterface() throws Exception { @Test public void testConstructorLeak_FailsToSetTTL() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try { new UdpLineChannel(FAILS_SET_SET_TTL_NET_FACADE, 1, 1, 9000, 10); fail("the channel should fail to instantiate when NF fails to set multicast interface"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java index edac6e9..a2065a4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java @@ -35,6 +35,7 @@ import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import java.util.concurrent.CountDownLatch; @@ -64,106 +65,108 @@ public class AsyncModeIntegrationTest { */ @Test public void testBackpressureBlocksEnqueueUntilAck() throws Exception { - InFlightWindow window = new InFlightWindow(2, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - AtomicLong highestSent = new AtomicLong(-1); - AtomicLong highestAcked = new AtomicLong(-1); - CountDownLatch twoSent = new CountDownLatch(2); - AtomicBoolean deliverAcks = new AtomicBoolean(false); - - client.setSendBehavior((ptr, len) -> { - highestSent.incrementAndGet(); - twoSent.countDown(); - }); - client.setTryReceiveBehavior(handler -> { - if (deliverAcks.get()) { - long sent = highestSent.get(); - long acked = highestAcked.get(); - if (sent > acked) { - highestAcked.set(sent); - emitAck(handler, sent); - return true; + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(2, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + CountDownLatch twoSent = new CountDownLatch(2); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + + client.setSendBehavior((ptr, len) -> { + highestSent.incrementAndGet(); + twoSent.countDown(); + }); + client.setTryReceiveBehavior(handler -> { + if (deliverAcks.get()) { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } } - } - return false; - }); + return false; + }); - WebSocketSendQueue queue = null; - MicrobatchBuffer buf0 = new MicrobatchBuffer(256); - MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); - try { - queue = new WebSocketSendQueue(client, window, 3_000, 500); - - // Send 2 batches to fill the window. - buf0.writeByte((byte) 1); - buf0.incrementRowCount(); - buf0.seal(); - queue.enqueue(buf0); - - buf1.writeByte((byte) 2); - buf1.incrementRowCount(); - buf1.seal(); - queue.enqueue(buf1); - - assertTrue("Both batches should be sent", twoSent.await(2, TimeUnit.SECONDS)); - assertEquals("Window should be full", 2, window.getInFlightCount()); - - // Reuse buf0 (recycled by I/O thread) and enqueue a 3rd batch. - // The I/O thread cannot poll it because the window is full. - assertTrue(buf0.awaitRecycled(2, TimeUnit.SECONDS)); - buf0.reset(); - buf0.writeByte((byte) 3); - buf0.incrementRowCount(); - buf0.seal(); - queue.enqueue(buf0); - - // Reuse buf1 and try to enqueue a 4th batch on a background - // thread. It should block because the pending slot is still - // occupied by the 3rd batch. - assertTrue(buf1.awaitRecycled(2, TimeUnit.SECONDS)); - buf1.reset(); - buf1.writeByte((byte) 4); - buf1.incrementRowCount(); - buf1.seal(); - - CountDownLatch enqueueStarted = new CountDownLatch(1); - CountDownLatch enqueueDone = new CountDownLatch(1); - AtomicReference errorRef = new AtomicReference<>(); - WebSocketSendQueue q = queue; - - Thread enqueueThread = new Thread(() -> { - enqueueStarted.countDown(); - try { - q.enqueue(buf1); - } catch (Throwable t) { - errorRef.set(t); - } finally { - enqueueDone.countDown(); - } - }); - enqueueThread.start(); + try { + queue = new WebSocketSendQueue(client, window, 3_000, 500); + + // Send 2 batches to fill the window. + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); + + assertTrue("Both batches should be sent", twoSent.await(2, TimeUnit.SECONDS)); + assertEquals("Window should be full", 2, window.getInFlightCount()); + + // Reuse buf0 (recycled by I/O thread) and enqueue a 3rd batch. + // The I/O thread cannot poll it because the window is full. + assertTrue(buf0.awaitRecycled(2, TimeUnit.SECONDS)); + buf0.reset(); + buf0.writeByte((byte) 3); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + // Reuse buf1 and try to enqueue a 4th batch on a background + // thread. It should block because the pending slot is still + // occupied by the 3rd batch. + assertTrue(buf1.awaitRecycled(2, TimeUnit.SECONDS)); + buf1.reset(); + buf1.writeByte((byte) 4); + buf1.incrementRowCount(); + buf1.seal(); + + CountDownLatch enqueueStarted = new CountDownLatch(1); + CountDownLatch enqueueDone = new CountDownLatch(1); + AtomicReference errorRef = new AtomicReference<>(); + WebSocketSendQueue q = queue; + + Thread enqueueThread = new Thread(() -> { + enqueueStarted.countDown(); + try { + q.enqueue(buf1); + } catch (Throwable t) { + errorRef.set(t); + } finally { + enqueueDone.countDown(); + } + }); + enqueueThread.start(); - assertTrue(enqueueStarted.await(1, TimeUnit.SECONDS)); - Thread.sleep(200); - assertEquals("Enqueue should still be blocked", 1, enqueueDone.getCount()); + assertTrue(enqueueStarted.await(1, TimeUnit.SECONDS)); + Thread.sleep(200); + assertEquals("Enqueue should still be blocked", 1, enqueueDone.getCount()); - // Deliver ACKs to unblock the pipeline. - deliverAcks.set(true); + // Deliver ACKs to unblock the pipeline. + deliverAcks.set(true); - assertTrue("Enqueue should complete after ACK", enqueueDone.await(3, TimeUnit.SECONDS)); - assertNull("No error expected", errorRef.get()); + assertTrue("Enqueue should complete after ACK", enqueueDone.await(3, TimeUnit.SECONDS)); + assertNull("No error expected", errorRef.get()); - queue.flush(); - window.awaitEmpty(); - } finally { - deliverAcks.set(true); - window.acknowledgeUpTo(Long.MAX_VALUE); - closeQuietly(queue); - buf0.close(); - buf1.close(); - client.close(); - } + queue.flush(); + window.awaitEmpty(); + } finally { + deliverAcks.set(true); + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); } /** @@ -173,67 +176,69 @@ public void testBackpressureBlocksEnqueueUntilAck() throws Exception { */ @Test public void testBatchesCycleThroughDoubleBuffers() throws Exception { - InFlightWindow window = new InFlightWindow(4, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - AtomicLong highestSent = new AtomicLong(-1); - AtomicLong highestAcked = new AtomicLong(-1); - - client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); - client.setTryReceiveBehavior(handler -> { - long sent = highestSent.get(); - long acked = highestAcked.get(); - if (sent > acked) { - highestAcked.set(sent); - emitAck(handler, sent); - return true; - } - return false; - }); + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + return false; + }); - WebSocketSendQueue queue = null; - MicrobatchBuffer buf0 = new MicrobatchBuffer(256); - MicrobatchBuffer buf1 = new MicrobatchBuffer(256); - int batchCount = 10; + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + int batchCount = 10; - try { - queue = new WebSocketSendQueue(client, window, 5_000, 500); - MicrobatchBuffer active = buf0; + try { + queue = new WebSocketSendQueue(client, window, 5_000, 500); + MicrobatchBuffer active = buf0; - for (int i = 0; i < batchCount; i++) { - if (active.isRecycled()) { - active.reset(); - } - assertTrue("Buffer should be FILLING on iteration " + i, active.isFilling()); - - active.writeByte((byte) (i & 0xFF)); - active.incrementRowCount(); - active.seal(); - queue.enqueue(active); - - // Swap to the other buffer, waiting for it if still in use. - MicrobatchBuffer other = (active == buf0) ? buf1 : buf0; - if (other.isInUse()) { - assertTrue("Other buffer should recycle", - other.awaitRecycled(2, TimeUnit.SECONDS)); - } - if (other.isRecycled()) { - other.reset(); + for (int i = 0; i < batchCount; i++) { + if (active.isRecycled()) { + active.reset(); + } + assertTrue("Buffer should be FILLING on iteration " + i, active.isFilling()); + + active.writeByte((byte) (i & 0xFF)); + active.incrementRowCount(); + active.seal(); + queue.enqueue(active); + + // Swap to the other buffer, waiting for it if still in use. + MicrobatchBuffer other = (active == buf0) ? buf1 : buf0; + if (other.isInUse()) { + assertTrue("Other buffer should recycle", + other.awaitRecycled(2, TimeUnit.SECONDS)); + } + if (other.isRecycled()) { + other.reset(); + } + active = other; } - active = other; - } - queue.flush(); - window.awaitEmpty(); + queue.flush(); + window.awaitEmpty(); - assertEquals(batchCount, queue.getTotalBatchesSent()); - assertEquals(0, window.getInFlightCount()); - } finally { - window.acknowledgeUpTo(Long.MAX_VALUE); - closeQuietly(queue); - buf0.close(); - buf1.close(); - client.close(); - } + assertEquals(batchCount, queue.getTotalBatchesSent()); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); } /** @@ -244,86 +249,88 @@ public void testBatchesCycleThroughDoubleBuffers() throws Exception { */ @Test public void testBufferSwapWaitsForSlowSend() throws Exception { - InFlightWindow window = new InFlightWindow(4, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - AtomicLong highestSent = new AtomicLong(-1); - AtomicLong highestAcked = new AtomicLong(-1); - CountDownLatch sendStarted = new CountDownLatch(1); - CountDownLatch sendGate = new CountDownLatch(1); - - client.setSendBehavior((ptr, len) -> { - long seq = highestSent.incrementAndGet(); - if (seq == 0) { - // Block on first send to simulate slow I/O. - sendStarted.countDown(); - try { - if (!sendGate.await(5, TimeUnit.SECONDS)) { - throw new RuntimeException("sendGate timed out"); + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); + CountDownLatch sendStarted = new CountDownLatch(1); + CountDownLatch sendGate = new CountDownLatch(1); + + client.setSendBehavior((ptr, len) -> { + long seq = highestSent.incrementAndGet(); + if (seq == 0) { + // Block on first send to simulate slow I/O. + sendStarted.countDown(); + try { + if (!sendGate.await(5, TimeUnit.SECONDS)) { + throw new RuntimeException("sendGate timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); } - } - }); - client.setTryReceiveBehavior(handler -> { - long sent = highestSent.get(); - long acked = highestAcked.get(); - if (sent > acked) { - highestAcked.set(sent); - emitAck(handler, sent); - return true; - } - return false; - }); + }); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + highestAcked.set(sent); + emitAck(handler, sent); + return true; + } + return false; + }); - WebSocketSendQueue queue = null; - MicrobatchBuffer buf0 = new MicrobatchBuffer(256); - MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); - try { - queue = new WebSocketSendQueue(client, window, 5_000, 500); + try { + queue = new WebSocketSendQueue(client, window, 5_000, 500); - // Enqueue buf0. The I/O thread starts sending and blocks. - buf0.writeByte((byte) 1); - buf0.incrementRowCount(); - buf0.seal(); - queue.enqueue(buf0); + // Enqueue buf0. The I/O thread starts sending and blocks. + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); - assertTrue("I/O thread should start sending", sendStarted.await(2, TimeUnit.SECONDS)); - assertTrue("buf0 should be in use (SENDING)", buf0.isInUse()); + assertTrue("I/O thread should start sending", sendStarted.await(2, TimeUnit.SECONDS)); + assertTrue("buf0 should be in use (SENDING)", buf0.isInUse()); - // Enqueue buf1 into the pending slot (I/O thread is blocked). - buf1.writeByte((byte) 2); - buf1.incrementRowCount(); - buf1.seal(); - queue.enqueue(buf1); + // Enqueue buf1 into the pending slot (I/O thread is blocked). + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); - // The user wants to reuse buf0, but it is still SENDING. - assertTrue("buf0 should still be in use", buf0.isInUse()); + // The user wants to reuse buf0, but it is still SENDING. + assertTrue("buf0 should still be in use", buf0.isInUse()); - // Release the gate so the I/O thread can finish sending buf0. - sendGate.countDown(); + // Release the gate so the I/O thread can finish sending buf0. + sendGate.countDown(); - // buf0 transitions SENDING -> RECYCLED. - assertTrue("buf0 should be recycled after send completes", - buf0.awaitRecycled(2, TimeUnit.SECONDS)); - assertTrue(buf0.isRecycled()); + // buf0 transitions SENDING -> RECYCLED. + assertTrue("buf0 should be recycled after send completes", + buf0.awaitRecycled(2, TimeUnit.SECONDS)); + assertTrue(buf0.isRecycled()); - // Reset and verify buf0 is reusable. - buf0.reset(); - assertTrue(buf0.isFilling()); + // Reset and verify buf0 is reusable. + buf0.reset(); + assertTrue(buf0.isFilling()); - queue.flush(); - window.awaitEmpty(); - assertEquals(2, queue.getTotalBatchesSent()); - } finally { - sendGate.countDown(); - window.acknowledgeUpTo(Long.MAX_VALUE); - closeQuietly(queue); - buf0.close(); - buf1.close(); - client.close(); - } + queue.flush(); + window.awaitEmpty(); + assertEquals(2, queue.getTotalBatchesSent()); + } finally { + sendGate.countDown(); + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); } /** @@ -334,49 +341,51 @@ public void testBufferSwapWaitsForSlowSend() throws Exception { */ @Test public void testFlushWaitsForSendButNotForAcks() throws Exception { - InFlightWindow window = new InFlightWindow(4, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - AtomicLong highestSent = new AtomicLong(-1); - AtomicBoolean deliverAcks = new AtomicBoolean(false); - - client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); - client.setTryReceiveBehavior(handler -> { - if (deliverAcks.get()) { - long sent = highestSent.get(); - if (sent >= 0 && window.getInFlightCount() > 0) { - emitAck(handler, sent); - return true; + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + if (deliverAcks.get()) { + long sent = highestSent.get(); + if (sent >= 0 && window.getInFlightCount() > 0) { + emitAck(handler, sent); + return true; + } } - } - return false; - }); + return false; + }); - WebSocketSendQueue queue = null; - MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); - try { - queue = new WebSocketSendQueue(client, window, 2_000, 500); - - buf0.writeByte((byte) 1); - buf0.incrementRowCount(); - buf0.seal(); - queue.enqueue(buf0); - - // flush() returns once the batch is sent, not when ACKed. - queue.flush(); - assertEquals(1, queue.getTotalBatchesSent()); - assertEquals("Batch should still be in flight", 1, window.getInFlightCount()); - - // Deliver ACK and wait for the window to drain. - deliverAcks.set(true); - window.awaitEmpty(); - assertEquals(0, window.getInFlightCount()); - } finally { - window.acknowledgeUpTo(Long.MAX_VALUE); - closeQuietly(queue); - buf0.close(); - client.close(); - } + try { + queue = new WebSocketSendQueue(client, window, 2_000, 500); + + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); + + // flush() returns once the batch is sent, not when ACKed. + queue.flush(); + assertEquals(1, queue.getTotalBatchesSent()); + assertEquals("Batch should still be in flight", 1, window.getInFlightCount()); + + // Deliver ACK and wait for the window to drain. + deliverAcks.set(true); + window.awaitEmpty(); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + client.close(); + } + }); } /** @@ -386,65 +395,67 @@ public void testFlushWaitsForSendButNotForAcks() throws Exception { */ @Test public void testHighThroughputWithManyBatches() throws Exception { - int batchCount = 50; - int windowSize = 4; - - InFlightWindow window = new InFlightWindow(windowSize, 10_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - AtomicLong highestSent = new AtomicLong(-1); - AtomicLong highestAcked = new AtomicLong(-1); - - client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); - client.setTryReceiveBehavior(handler -> { - long sent = highestSent.get(); - long acked = highestAcked.get(); - if (sent > acked) { - // ACK one batch at a time to test sustained flow. - long next = acked + 1; - highestAcked.set(next); - emitAck(handler, next); - return true; - } - return false; - }); + assertMemoryLeak(() -> { + int batchCount = 50; + int windowSize = 4; - WebSocketSendQueue queue = null; - MicrobatchBuffer buf0 = new MicrobatchBuffer(256); - MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + InFlightWindow window = new InFlightWindow(windowSize, 10_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestAcked = new AtomicLong(-1); - try { - queue = new WebSocketSendQueue(client, window, 10_000, 2_000); - MicrobatchBuffer active = buf0; - - for (int i = 0; i < batchCount; i++) { - if (!active.isFilling()) { - if (active.isInUse()) { - assertTrue("Buffer should recycle on iteration " + i, - active.awaitRecycled(5, TimeUnit.SECONDS)); - } - active.reset(); + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long acked = highestAcked.get(); + if (sent > acked) { + // ACK one batch at a time to test sustained flow. + long next = acked + 1; + highestAcked.set(next); + emitAck(handler, next); + return true; } + return false; + }); - active.writeByte((byte) (i & 0xFF)); - active.incrementRowCount(); - active.seal(); - queue.enqueue(active); + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); - active = (active == buf0) ? buf1 : buf0; - } + try { + queue = new WebSocketSendQueue(client, window, 10_000, 2_000); + MicrobatchBuffer active = buf0; + + for (int i = 0; i < batchCount; i++) { + if (!active.isFilling()) { + if (active.isInUse()) { + assertTrue("Buffer should recycle on iteration " + i, + active.awaitRecycled(5, TimeUnit.SECONDS)); + } + active.reset(); + } - queue.flush(); - window.awaitEmpty(); + active.writeByte((byte) (i & 0xFF)); + active.incrementRowCount(); + active.seal(); + queue.enqueue(active); - assertEquals(batchCount, queue.getTotalBatchesSent()); - assertEquals(0, window.getInFlightCount()); - } finally { - window.acknowledgeUpTo(Long.MAX_VALUE); - closeQuietly(queue); - buf0.close(); - buf1.close(); - client.close(); - } + active = (active == buf0) ? buf1 : buf0; + } + + queue.flush(); + window.awaitEmpty(); + + assertEquals(batchCount, queue.getTotalBatchesSent()); + assertEquals(0, window.getInFlightCount()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); + } + }); } /** @@ -454,62 +465,64 @@ public void testHighThroughputWithManyBatches() throws Exception { */ @Test public void testServerErrorPropagatesOnFlush() throws Exception { - InFlightWindow window = new InFlightWindow(4, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - AtomicLong highestSent = new AtomicLong(-1); - AtomicLong highestDelivered = new AtomicLong(-1); - - client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); - client.setTryReceiveBehavior(handler -> { - long sent = highestSent.get(); - long delivered = highestDelivered.get(); - if (sent > delivered) { - long next = delivered + 1; - highestDelivered.set(next); - if (next == 1) { - emitError(handler, next, WebSocketResponse.STATUS_WRITE_ERROR, "disk full"); - } else { - emitAck(handler, next); + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(4, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicLong highestDelivered = new AtomicLong(-1); + + client.setSendBehavior((ptr, len) -> highestSent.incrementAndGet()); + client.setTryReceiveBehavior(handler -> { + long sent = highestSent.get(); + long delivered = highestDelivered.get(); + if (sent > delivered) { + long next = delivered + 1; + highestDelivered.set(next); + if (next == 1) { + emitError(handler, next, WebSocketResponse.STATUS_WRITE_ERROR, "disk full"); + } else { + emitAck(handler, next); + } + return true; } - return true; - } - return false; - }); + return false; + }); - WebSocketSendQueue queue = null; - MicrobatchBuffer buf0 = new MicrobatchBuffer(256); - MicrobatchBuffer buf1 = new MicrobatchBuffer(256); + WebSocketSendQueue queue = null; + MicrobatchBuffer buf0 = new MicrobatchBuffer(256); + MicrobatchBuffer buf1 = new MicrobatchBuffer(256); - try { - queue = new WebSocketSendQueue(client, window, 2_000, 500); + try { + queue = new WebSocketSendQueue(client, window, 2_000, 500); - buf0.writeByte((byte) 1); - buf0.incrementRowCount(); - buf0.seal(); - queue.enqueue(buf0); + buf0.writeByte((byte) 1); + buf0.incrementRowCount(); + buf0.seal(); + queue.enqueue(buf0); - buf1.writeByte((byte) 2); - buf1.incrementRowCount(); - buf1.seal(); - queue.enqueue(buf1); + buf1.writeByte((byte) 2); + buf1.incrementRowCount(); + buf1.seal(); + queue.enqueue(buf1); - // flush() waits for the queue to drain (both batches sent). - queue.flush(); + // flush() waits for the queue to drain (both batches sent). + queue.flush(); - // awaitEmpty() surfaces the server error for batch 1. - try { - window.awaitEmpty(); - fail("Expected server error to propagate"); - } catch (LineSenderException e) { - assertTrue("Error should mention server failure", - e.getMessage().contains("disk full") || e.getMessage().contains("Server error")); + // awaitEmpty() surfaces the server error for batch 1. + try { + window.awaitEmpty(); + fail("Expected server error to propagate"); + } catch (LineSenderException e) { + assertTrue("Error should mention server failure", + e.getMessage().contains("disk full") || e.getMessage().contains("Server error")); + } + } finally { + closeQuietly(queue); + buf0.close(); + buf1.close(); + client.close(); } - } finally { - closeQuietly(queue); - buf0.close(); - buf1.close(); - client.close(); - } + }); } private static void closeQuietly(WebSocketSendQueue queue) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java index 7001e84..f24dc35 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java @@ -30,6 +30,7 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -48,494 +49,542 @@ public class DeltaSymbolDictionaryTest { @Test - public void testEdgeCase_batchWithNoSymbols() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + public void testEdgeCase_batchWithNoSymbols() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table with only non-symbol columns + try (QwpTableBuffer batch = new QwpTableBuffer("metrics")) { + QwpTableBuffer.ColumnBuffer valueCol = batch.getOrCreateColumn("value", TYPE_LONG, false); + valueCol.addLong(100L); + batch.nextRow(); + + // MaxId is -1 (no symbols) + int batchMaxId = -1; + + // Can still encode with delta dict (empty delta) + int size = encoder.encodeWithDeltaDict(batch, globalDict, -1, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify flag is set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + } + }); + } + + @Test + public void testEdgeCase_duplicateSymbolsInBatch() throws Exception { + assertMemoryLeak(() -> { GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - // Table with only non-symbol columns - QwpTableBuffer batch = new QwpTableBuffer("metrics"); - QwpTableBuffer.ColumnBuffer valueCol = batch.getOrCreateColumn("value", TYPE_LONG, false); - valueCol.addLong(100L); - batch.nextRow(); + try (QwpTableBuffer batch = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); - // MaxId is -1 (no symbols) - int batchMaxId = -1; + // Same symbol used multiple times + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); - // Can still encode with delta dict (empty delta) - int size = encoder.encodeWithDeltaDict(batch, globalDict, -1, batchMaxId, false); - Assert.assertTrue(size > 0); + Assert.assertEquals(3, batch.getRowCount()); + Assert.assertEquals(1, globalDict.size()); // Only 1 unique symbol - // Verify flag is set - QwpBufferWriter buf = encoder.getBuffer(); - byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); - Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); - } + int maxGlobalId = col.getMaxGlobalSymbolId(); + Assert.assertEquals(0, maxGlobalId); // Max ID is 0 (AAPL) + } + }); } @Test - public void testEdgeCase_duplicateSymbolsInBatch() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - - QwpTableBuffer batch = new QwpTableBuffer("test"); - QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); - - // Same symbol used multiple times - int aaplId = globalDict.getOrAddSymbol("AAPL"); - col.addSymbolWithGlobalId("AAPL", aaplId); - batch.nextRow(); - col.addSymbolWithGlobalId("AAPL", aaplId); - batch.nextRow(); - col.addSymbolWithGlobalId("AAPL", aaplId); - batch.nextRow(); - - Assert.assertEquals(3, batch.getRowCount()); - Assert.assertEquals(1, globalDict.size()); // Only 1 unique symbol - - int maxGlobalId = col.getMaxGlobalSymbolId(); - Assert.assertEquals(0, maxGlobalId); // Max ID is 0 (AAPL) - } + public void testEdgeCase_emptyBatch() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - @Test - public void testEdgeCase_emptyBatch() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - - // Pre-populate dictionary and send - globalDict.getOrAddSymbol("AAPL"); - int maxSentSymbolId = 0; - - // Empty batch (no rows, no symbols used) - QwpTableBuffer emptyBatch = new QwpTableBuffer("test"); - Assert.assertEquals(0, emptyBatch.getRowCount()); - - // Delta should still work (deltaCount = 0) - int deltaStart = maxSentSymbolId + 1; - int deltaCount = 0; - Assert.assertEquals(1, deltaStart); - Assert.assertEquals(0, deltaCount); - } + // Pre-populate dictionary and send + globalDict.getOrAddSymbol("AAPL"); + int maxSentSymbolId = 0; - @Test - public void testEdgeCase_gapFill() { - // Client dictionary: AAPL(0), GOOG(1), MSFT(2), TSLA(3) - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - globalDict.getOrAddSymbol("AAPL"); - globalDict.getOrAddSymbol("GOOG"); - globalDict.getOrAddSymbol("MSFT"); - globalDict.getOrAddSymbol("TSLA"); - - // Batch uses AAPL(0) and TSLA(3), skipping GOOG(1) and MSFT(2) - // Delta must include gap-fill: send all symbols from maxSentSymbolId+1 to batchMaxId - int maxSentSymbolId = -1; - int batchMaxId = 3; // TSLA - - int deltaStart = maxSentSymbolId + 1; - int deltaCount = batchMaxId - maxSentSymbolId; - - // Must send symbols 0, 1, 2, 3 (even though 1, 2 aren't used in this batch) - Assert.assertEquals(0, deltaStart); - Assert.assertEquals(4, deltaCount); - - // This ensures server has contiguous dictionary - for (int id = deltaStart; id < deltaStart + deltaCount; id++) { - String symbol = globalDict.getSymbol(id); - Assert.assertNotNull("Symbol " + id + " should exist", symbol); - } + // Empty batch (no rows, no symbols used) + try (QwpTableBuffer emptyBatch = new QwpTableBuffer("test")) { + Assert.assertEquals(0, emptyBatch.getRowCount()); + + // Delta should still work (deltaCount = 0) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 0; + Assert.assertEquals(1, deltaStart); + Assert.assertEquals(0, deltaCount); + } + }); } @Test - public void testEdgeCase_largeSymbolDictionary() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + public void testEdgeCase_gapFill() throws Exception { + assertMemoryLeak(() -> { + // Client dictionary: AAPL(0), GOOG(1), MSFT(2), TSLA(3) + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); + globalDict.getOrAddSymbol("TSLA"); - // Add 1000 unique symbols - for (int i = 0; i < 1000; i++) { - int id = globalDict.getOrAddSymbol("SYM_" + i); - Assert.assertEquals(i, id); - } + // Batch uses AAPL(0) and TSLA(3), skipping GOOG(1) and MSFT(2) + // Delta must include gap-fill: send all symbols from maxSentSymbolId+1 to batchMaxId + int maxSentSymbolId = -1; + int batchMaxId = 3; // TSLA - Assert.assertEquals(1000, globalDict.size()); + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batchMaxId - maxSentSymbolId; - // Send first batch with symbols 0-99 - int maxSentSymbolId = 99; + // Must send symbols 0, 1, 2, 3 (even though 1, 2 aren't used in this batch) + Assert.assertEquals(0, deltaStart); + Assert.assertEquals(4, deltaCount); - // Next batch uses symbols 0-199, delta is 100-199 - int deltaStart = maxSentSymbolId + 1; - int deltaCount = 199 - maxSentSymbolId; - Assert.assertEquals(100, deltaStart); - Assert.assertEquals(100, deltaCount); + // This ensures server has contiguous dictionary + for (int id = deltaStart; id < deltaStart + deltaCount; id++) { + String symbol = globalDict.getSymbol(id); + Assert.assertNotNull("Symbol " + id + " should exist", symbol); + } + }); } @Test - public void testEdgeCase_nullSymbolValues() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - - QwpTableBuffer batch = new QwpTableBuffer("test"); - QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, true); // nullable + public void testEdgeCase_largeSymbolDictionary() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - int aaplId = globalDict.getOrAddSymbol("AAPL"); - col.addSymbolWithGlobalId("AAPL", aaplId); - batch.nextRow(); + // Add 1000 unique symbols + for (int i = 0; i < 1000; i++) { + int id = globalDict.getOrAddSymbol("SYM_" + i); + Assert.assertEquals(i, id); + } - col.addSymbol(null); // NULL value - batch.nextRow(); + Assert.assertEquals(1000, globalDict.size()); - col.addSymbolWithGlobalId("AAPL", aaplId); - batch.nextRow(); + // Send first batch with symbols 0-99 + int maxSentSymbolId = 99; - Assert.assertEquals(3, batch.getRowCount()); - // Dictionary only has 1 symbol (AAPL), NULL doesn't add to dictionary - Assert.assertEquals(1, globalDict.size()); + // Next batch uses symbols 0-199, delta is 100-199 + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 199 - maxSentSymbolId; + Assert.assertEquals(100, deltaStart); + Assert.assertEquals(100, deltaCount); + }); } @Test - public void testEdgeCase_unicodeSymbols() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - - // Unicode symbols - int id1 = globalDict.getOrAddSymbol("日本"); - int id2 = globalDict.getOrAddSymbol("中国"); - int id3 = globalDict.getOrAddSymbol("한국"); - int id4 = globalDict.getOrAddSymbol("Émoji🚀"); - - Assert.assertEquals(0, id1); - Assert.assertEquals(1, id2); - Assert.assertEquals(2, id3); - Assert.assertEquals(3, id4); - - Assert.assertEquals("日本", globalDict.getSymbol(0)); - Assert.assertEquals("中国", globalDict.getSymbol(1)); - Assert.assertEquals("한국", globalDict.getSymbol(2)); - Assert.assertEquals("Émoji🚀", globalDict.getSymbol(3)); - } + public void testEdgeCase_nullSymbolValues() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - @Test - public void testEdgeCase_veryLongSymbol() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + try (QwpTableBuffer batch = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, true); // nullable - // Create a very long symbol (1000 chars) - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < 1000; i++) { - sb.append('X'); - } - String longSymbol = sb.toString(); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + col.addSymbol(null); // NULL value + batch.nextRow(); - int id = globalDict.getOrAddSymbol(longSymbol); - Assert.assertEquals(0, id); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); - String retrieved = globalDict.getSymbol(0); - Assert.assertEquals(longSymbol, retrieved); - Assert.assertEquals(1000, retrieved.length()); + Assert.assertEquals(3, batch.getRowCount()); + // Dictionary only has 1 symbol (AAPL), NULL doesn't add to dictionary + Assert.assertEquals(1, globalDict.size()); + } + }); } @Test - public void testMultipleBatches_encodeAndDecode() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); - ObjList serverDict = new ObjList<>(); - int maxSentSymbolId = -1; - QwpTableBuffer batch1 = new QwpTableBuffer("test"); - QwpTableBuffer.ColumnBuffer col1 = batch1.getOrCreateColumn("sym", TYPE_SYMBOL, false); - - int aaplId = clientDict.getOrAddSymbol("AAPL"); - int googId = clientDict.getOrAddSymbol("GOOG"); - col1.addSymbolWithGlobalId("AAPL", aaplId); - batch1.nextRow(); - col1.addSymbolWithGlobalId("GOOG", googId); - batch1.nextRow(); - - int batch1MaxId = 1; - int size1 = encoder.encodeWithDeltaDict(batch1, clientDict, maxSentSymbolId, batch1MaxId, false); - Assert.assertTrue(size1 > 0); - maxSentSymbolId = batch1MaxId; - - // Decode on server side - QwpBufferWriter buf1 = encoder.getBuffer(); - decodeAndAccumulateDict(buf1.getBufferPtr(), size1, serverDict); - - // Verify server dictionary - Assert.assertEquals(2, serverDict.size()); - Assert.assertEquals("AAPL", serverDict.get(0)); - Assert.assertEquals("GOOG", serverDict.get(1)); - QwpTableBuffer batch2 = new QwpTableBuffer("test"); - QwpTableBuffer.ColumnBuffer col2 = batch2.getOrCreateColumn("sym", TYPE_SYMBOL, false); + public void testEdgeCase_unicodeSymbols() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Unicode symbols + int id1 = globalDict.getOrAddSymbol("日本"); + int id2 = globalDict.getOrAddSymbol("中国"); + int id3 = globalDict.getOrAddSymbol("한국"); + int id4 = globalDict.getOrAddSymbol("Émoji🚀"); + + Assert.assertEquals(0, id1); + Assert.assertEquals(1, id2); + Assert.assertEquals(2, id3); + Assert.assertEquals(3, id4); + + Assert.assertEquals("日本", globalDict.getSymbol(0)); + Assert.assertEquals("中国", globalDict.getSymbol(1)); + Assert.assertEquals("한국", globalDict.getSymbol(2)); + Assert.assertEquals("Émoji🚀", globalDict.getSymbol(3)); + }); + } - int msftId = clientDict.getOrAddSymbol("MSFT"); - col2.addSymbolWithGlobalId("AAPL", aaplId); // Existing - batch2.nextRow(); - col2.addSymbolWithGlobalId("MSFT", msftId); // New - batch2.nextRow(); + @Test + public void testEdgeCase_veryLongSymbol() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - int batch2MaxId = 2; - int size2 = encoder.encodeWithDeltaDict(batch2, clientDict, maxSentSymbolId, batch2MaxId, false); - Assert.assertTrue(size2 > 0); - maxSentSymbolId = batch2MaxId; + // Create a very long symbol (1000 chars) + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + sb.append('X'); + } + String longSymbol = sb.toString(); - // Decode batch 2 - QwpBufferWriter buf2 = encoder.getBuffer(); - decodeAndAccumulateDict(buf2.getBufferPtr(), size2, serverDict); + int id = globalDict.getOrAddSymbol(longSymbol); + Assert.assertEquals(0, id); - // Server dictionary should now have 3 symbols - Assert.assertEquals(3, serverDict.size()); - Assert.assertEquals("MSFT", serverDict.get(2)); - } + String retrieved = globalDict.getSymbol(0); + Assert.assertEquals(longSymbol, retrieved); + Assert.assertEquals(1000, retrieved.length()); + }); } @Test - public void testMultipleBatches_progressiveSymbolAccumulation() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - - // Batch 1: AAPL, GOOG - int aaplId = globalDict.getOrAddSymbol("AAPL"); - int googId = globalDict.getOrAddSymbol("GOOG"); - int batch1MaxId = Math.max(aaplId, googId); - - // Simulate sending batch 1 - maxSentSymbolId = 1 after send - int maxSentSymbolId = batch1MaxId; // 1 - - // Batch 2: AAPL (existing), MSFT (new), TSLA (new) - globalDict.getOrAddSymbol("AAPL"); // Returns 0, already exists - int msftId = globalDict.getOrAddSymbol("MSFT"); - int tslaId = globalDict.getOrAddSymbol("TSLA"); - int batch2MaxId = Math.max(msftId, tslaId); - - // Delta for batch 2 should be [2, 3] (MSFT, TSLA) - int deltaStart = maxSentSymbolId + 1; - int deltaCount = batch2MaxId - maxSentSymbolId; - Assert.assertEquals(2, deltaStart); - Assert.assertEquals(2, deltaCount); - - // Simulate sending batch 2 - maxSentSymbolId = batch2MaxId; // 3 - - // Batch 3: All existing symbols (no delta needed) - globalDict.getOrAddSymbol("AAPL"); - globalDict.getOrAddSymbol("GOOG"); - int batch3MaxId = 1; // Max used is GOOG(1) - - deltaStart = maxSentSymbolId + 1; - deltaCount = Math.max(0, batch3MaxId - maxSentSymbolId); - Assert.assertEquals(4, deltaStart); - Assert.assertEquals(0, deltaCount); // No new symbols + public void testMultipleBatches_encodeAndDecode() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + ObjList serverDict = new ObjList<>(); + int maxSentSymbolId = -1; + + int aaplId = clientDict.getOrAddSymbol("AAPL"); + int googId = clientDict.getOrAddSymbol("GOOG"); + + try (QwpTableBuffer batch1 = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col1 = batch1.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + col1.addSymbolWithGlobalId("AAPL", aaplId); + batch1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + batch1.nextRow(); + + int batch1MaxId = 1; + int size1 = encoder.encodeWithDeltaDict(batch1, clientDict, maxSentSymbolId, batch1MaxId, false); + Assert.assertTrue(size1 > 0); + maxSentSymbolId = batch1MaxId; + + // Decode on server side + QwpBufferWriter buf1 = encoder.getBuffer(); + decodeAndAccumulateDict(buf1.getBufferPtr(), size1, serverDict); + + // Verify server dictionary + Assert.assertEquals(2, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + } + + try (QwpTableBuffer batch2 = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col2 = batch2.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + int msftId = clientDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Existing + batch2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); // New + batch2.nextRow(); + + int batch2MaxId = 2; + int size2 = encoder.encodeWithDeltaDict(batch2, clientDict, maxSentSymbolId, batch2MaxId, false); + Assert.assertTrue(size2 > 0); + maxSentSymbolId = batch2MaxId; + + // Decode batch 2 + QwpBufferWriter buf2 = encoder.getBuffer(); + decodeAndAccumulateDict(buf2.getBufferPtr(), size2, serverDict); + + // Server dictionary should now have 3 symbols + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(2)); + } + } + }); } @Test - public void testMultipleTables_encodedInSameBatch() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + public void testMultipleBatches_progressiveSymbolAccumulation() throws Exception { + assertMemoryLeak(() -> { GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - // Create two tables - QwpTableBuffer table1 = new QwpTableBuffer("trades"); - QwpTableBuffer table2 = new QwpTableBuffer("quotes"); - - // Table 1: ticker column - QwpTableBuffer.ColumnBuffer col1 = table1.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + // Batch 1: AAPL, GOOG int aaplId = globalDict.getOrAddSymbol("AAPL"); int googId = globalDict.getOrAddSymbol("GOOG"); - col1.addSymbolWithGlobalId("AAPL", aaplId); - table1.nextRow(); - col1.addSymbolWithGlobalId("GOOG", googId); - table1.nextRow(); - - // Table 2: symbol column (different name, but shares dictionary) - QwpTableBuffer.ColumnBuffer col2 = table2.getOrCreateColumn("symbol", TYPE_SYMBOL, false); - int msftId = globalDict.getOrAddSymbol("MSFT"); - col2.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL - table2.nextRow(); - col2.addSymbolWithGlobalId("MSFT", msftId); - table2.nextRow(); + int batch1MaxId = Math.max(aaplId, googId); - // Encode first table with delta dict - int confirmedMaxId = -1; - int batchMaxId = 2; // AAPL(0), GOOG(1), MSFT(2) + // Simulate sending batch 1 - maxSentSymbolId = 1 after send + int maxSentSymbolId = batch1MaxId; // 1 - int size = encoder.encodeWithDeltaDict(table1, globalDict, confirmedMaxId, batchMaxId, false); - Assert.assertTrue(size > 0); - - // Verify delta section contains all 3 symbols - QwpBufferWriter buf = encoder.getBuffer(); - long ptr = buf.getBufferPtr(); - - byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); - Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + // Batch 2: AAPL (existing), MSFT (new), TSLA (new) + globalDict.getOrAddSymbol("AAPL"); // Returns 0, already exists + int msftId = globalDict.getOrAddSymbol("MSFT"); + int tslaId = globalDict.getOrAddSymbol("TSLA"); + int batch2MaxId = Math.max(msftId, tslaId); + + // Delta for batch 2 should be [2, 3] (MSFT, TSLA) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batch2MaxId - maxSentSymbolId; + Assert.assertEquals(2, deltaStart); + Assert.assertEquals(2, deltaCount); + + // Simulate sending batch 2 + maxSentSymbolId = batch2MaxId; // 3 + + // Batch 3: All existing symbols (no delta needed) + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + int batch3MaxId = 1; // Max used is GOOG(1) + + deltaStart = maxSentSymbolId + 1; + deltaCount = Math.max(0, batch3MaxId - maxSentSymbolId); + Assert.assertEquals(4, deltaStart); + Assert.assertEquals(0, deltaCount); // No new symbols + }); + } - // After header: deltaStart=0, deltaCount=3 - long pos = ptr + HEADER_SIZE; - int deltaStart = readVarint(pos); - Assert.assertEquals(0, deltaStart); - } + @Test + public void testMultipleTables_encodedInSameBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Create two tables + try (QwpTableBuffer table1 = new QwpTableBuffer("trades"); + QwpTableBuffer table2 = new QwpTableBuffer("quotes")) { + + // Table 1: ticker column + QwpTableBuffer.ColumnBuffer col1 = table1.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + col1.addSymbolWithGlobalId("AAPL", aaplId); + table1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + table1.nextRow(); + + // Table 2: symbol column (different name, but shares dictionary) + QwpTableBuffer.ColumnBuffer col2 = table2.getOrCreateColumn("symbol", TYPE_SYMBOL, false); + int msftId = globalDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); + table2.nextRow(); + + // Encode first table with delta dict + int confirmedMaxId = -1; + int batchMaxId = 2; // AAPL(0), GOOG(1), MSFT(2) + + int size = encoder.encodeWithDeltaDict(table1, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify delta section contains all 3 symbols + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // After header: deltaStart=0, deltaCount=3 + long pos = ptr + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + }); } @Test - public void testMultipleTables_multipleSymbolColumns() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - - QwpTableBuffer table = new QwpTableBuffer("market_data"); - - // Column 1: exchange - QwpTableBuffer.ColumnBuffer exchangeCol = table.getOrCreateColumn("exchange", TYPE_SYMBOL, false); - int nyseId = globalDict.getOrAddSymbol("NYSE"); - int nasdaqId = globalDict.getOrAddSymbol("NASDAQ"); - - // Column 2: currency - QwpTableBuffer.ColumnBuffer currencyCol = table.getOrCreateColumn("currency", TYPE_SYMBOL, false); - int usdId = globalDict.getOrAddSymbol("USD"); - int eurId = globalDict.getOrAddSymbol("EUR"); - - // Column 3: ticker - QwpTableBuffer.ColumnBuffer tickerCol = table.getOrCreateColumn("ticker", TYPE_SYMBOL, false); - int aaplId = globalDict.getOrAddSymbol("AAPL"); - - // Add row with all three columns - exchangeCol.addSymbolWithGlobalId("NYSE", nyseId); - currencyCol.addSymbolWithGlobalId("USD", usdId); - tickerCol.addSymbolWithGlobalId("AAPL", aaplId); - table.nextRow(); - - exchangeCol.addSymbolWithGlobalId("NASDAQ", nasdaqId); - currencyCol.addSymbolWithGlobalId("EUR", eurId); - tickerCol.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL - table.nextRow(); - - // All symbols share the same global dictionary - Assert.assertEquals(5, globalDict.size()); - Assert.assertEquals("NYSE", globalDict.getSymbol(0)); - Assert.assertEquals("NASDAQ", globalDict.getSymbol(1)); - Assert.assertEquals("USD", globalDict.getSymbol(2)); - Assert.assertEquals("EUR", globalDict.getSymbol(3)); - Assert.assertEquals("AAPL", globalDict.getSymbol(4)); + public void testMultipleTables_multipleSymbolColumns() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + try (QwpTableBuffer table = new QwpTableBuffer("market_data")) { + + // Column 1: exchange + QwpTableBuffer.ColumnBuffer exchangeCol = table.getOrCreateColumn("exchange", TYPE_SYMBOL, false); + int nyseId = globalDict.getOrAddSymbol("NYSE"); + int nasdaqId = globalDict.getOrAddSymbol("NASDAQ"); + + // Column 2: currency + QwpTableBuffer.ColumnBuffer currencyCol = table.getOrCreateColumn("currency", TYPE_SYMBOL, false); + int usdId = globalDict.getOrAddSymbol("USD"); + int eurId = globalDict.getOrAddSymbol("EUR"); + + // Column 3: ticker + QwpTableBuffer.ColumnBuffer tickerCol = table.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + + // Add row with all three columns + exchangeCol.addSymbolWithGlobalId("NYSE", nyseId); + currencyCol.addSymbolWithGlobalId("USD", usdId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); + table.nextRow(); + + exchangeCol.addSymbolWithGlobalId("NASDAQ", nasdaqId); + currencyCol.addSymbolWithGlobalId("EUR", eurId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table.nextRow(); + + // All symbols share the same global dictionary + Assert.assertEquals(5, globalDict.size()); + Assert.assertEquals("NYSE", globalDict.getSymbol(0)); + Assert.assertEquals("NASDAQ", globalDict.getSymbol(1)); + Assert.assertEquals("USD", globalDict.getSymbol(2)); + Assert.assertEquals("EUR", globalDict.getSymbol(3)); + Assert.assertEquals("AAPL", globalDict.getSymbol(4)); + } + }); } @Test - public void testMultipleTables_sharedGlobalDictionary() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + public void testMultipleTables_sharedGlobalDictionary() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table 1 uses symbols AAPL, GOOG + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); - // Table 1 uses symbols AAPL, GOOG - int aaplId = globalDict.getOrAddSymbol("AAPL"); - int googId = globalDict.getOrAddSymbol("GOOG"); + // Table 2 uses symbols AAPL (reused), MSFT (new) + int aaplId2 = globalDict.getOrAddSymbol("AAPL"); // Should return same ID + int msftId = globalDict.getOrAddSymbol("MSFT"); - // Table 2 uses symbols AAPL (reused), MSFT (new) - int aaplId2 = globalDict.getOrAddSymbol("AAPL"); // Should return same ID - int msftId = globalDict.getOrAddSymbol("MSFT"); + // Verify deduplication + Assert.assertEquals(0, aaplId); + Assert.assertEquals(1, googId); + Assert.assertEquals(0, aaplId2); // Same as aaplId + Assert.assertEquals(2, msftId); - // Verify deduplication - Assert.assertEquals(0, aaplId); - Assert.assertEquals(1, googId); - Assert.assertEquals(0, aaplId2); // Same as aaplId - Assert.assertEquals(2, msftId); + // Total symbols should be 3 + Assert.assertEquals(3, globalDict.size()); + }); + } - // Total symbols should be 3 - Assert.assertEquals(3, globalDict.size()); + @Test + public void testReconnection_fullDeltaAfterReconnect() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + + // First connection: add symbols + int aaplId = clientDict.getOrAddSymbol("AAPL"); + clientDict.getOrAddSymbol("GOOG"); + + // Send batch - maxSentSymbolId = 1 + int maxSentSymbolId = 1; + + // Reconnect - reset maxSentSymbolId + maxSentSymbolId = -1; + + // Create new batch using existing symbols + try (QwpTableBuffer batch = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + // Encode - should send full delta (all symbols from 0) + int size = encoder.encodeWithDeltaDict(batch, clientDict, maxSentSymbolId, 1, false); + Assert.assertTrue(size > 0); + + // Verify deltaStart is 0 + QwpBufferWriter buf = encoder.getBuffer(); + long pos = buf.getBufferPtr() + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + }); } @Test - public void testReconnection_fullDeltaAfterReconnect() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + public void testReconnection_resetsWatermark() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - // First connection: add symbols - int aaplId = clientDict.getOrAddSymbol("AAPL"); - clientDict.getOrAddSymbol("GOOG"); + // Build up dictionary and "send" some symbols + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); - // Send batch - maxSentSymbolId = 1 - int maxSentSymbolId = 1; + int maxSentSymbolId = 2; - // Reconnect - reset maxSentSymbolId + // Simulate reconnection - reset maxSentSymbolId maxSentSymbolId = -1; + Assert.assertEquals(-1, maxSentSymbolId); - // Create new batch using existing symbols - QwpTableBuffer batch = new QwpTableBuffer("test"); - QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); - col.addSymbolWithGlobalId("AAPL", aaplId); - batch.nextRow(); - - // Encode - should send full delta (all symbols from 0) - int size = encoder.encodeWithDeltaDict(batch, clientDict, maxSentSymbolId, 1, false); - Assert.assertTrue(size > 0); + // Global dictionary is NOT cleared (it's client-side) + Assert.assertEquals(3, globalDict.size()); - // Verify deltaStart is 0 - QwpBufferWriter buf = encoder.getBuffer(); - long pos = buf.getBufferPtr() + HEADER_SIZE; - int deltaStart = readVarint(pos); + // Next batch must send full delta from 0 + int deltaStart = maxSentSymbolId + 1; Assert.assertEquals(0, deltaStart); - } + }); } @Test - public void testReconnection_resetsWatermark() { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - - // Build up dictionary and "send" some symbols - globalDict.getOrAddSymbol("AAPL"); - globalDict.getOrAddSymbol("GOOG"); - globalDict.getOrAddSymbol("MSFT"); - - int maxSentSymbolId = 2; + public void testReconnection_serverDictionaryCleared() throws Exception { + assertMemoryLeak(() -> { + ObjList serverDict = new ObjList<>(); - // Simulate reconnection - reset maxSentSymbolId - maxSentSymbolId = -1; - Assert.assertEquals(-1, maxSentSymbolId); + // Simulate first connection + serverDict.add("AAPL"); + serverDict.add("GOOG"); + Assert.assertEquals(2, serverDict.size()); - // Global dictionary is NOT cleared (it's client-side) - Assert.assertEquals(3, globalDict.size()); + // Simulate reconnection - server clears dictionary + serverDict.clear(); + Assert.assertEquals(0, serverDict.size()); - // Next batch must send full delta from 0 - int deltaStart = maxSentSymbolId + 1; - Assert.assertEquals(0, deltaStart); - } - - @Test - public void testReconnection_serverDictionaryCleared() { - ObjList serverDict = new ObjList<>(); - - // Simulate first connection - serverDict.add("AAPL"); - serverDict.add("GOOG"); - Assert.assertEquals(2, serverDict.size()); - - // Simulate reconnection - server clears dictionary - serverDict.clear(); - Assert.assertEquals(0, serverDict.size()); - - // New connection starts fresh - serverDict.add("MSFT"); - Assert.assertEquals(1, serverDict.size()); - Assert.assertEquals("MSFT", serverDict.get(0)); + // New connection starts fresh + serverDict.add("MSFT"); + Assert.assertEquals(1, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(0)); + }); } @Test - public void testServerSide_accumulateDelta() { - ObjList serverDict = new ObjList<>(); + public void testServerSide_accumulateDelta() throws Exception { + assertMemoryLeak(() -> { + ObjList serverDict = new ObjList<>(); - // First batch: symbols 0-2 - accumulateDelta(serverDict, 0, new String[]{"AAPL", "GOOG", "MSFT"}); + // First batch: symbols 0-2 + accumulateDelta(serverDict, 0, new String[]{"AAPL", "GOOG", "MSFT"}); - Assert.assertEquals(3, serverDict.size()); - Assert.assertEquals("AAPL", serverDict.get(0)); - Assert.assertEquals("GOOG", serverDict.get(1)); - Assert.assertEquals("MSFT", serverDict.get(2)); + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); - // Second batch: symbols 3-4 - accumulateDelta(serverDict, 3, new String[]{"TSLA", "AMZN"}); + // Second batch: symbols 3-4 + accumulateDelta(serverDict, 3, new String[]{"TSLA", "AMZN"}); - Assert.assertEquals(5, serverDict.size()); - Assert.assertEquals("TSLA", serverDict.get(3)); - Assert.assertEquals("AMZN", serverDict.get(4)); + Assert.assertEquals(5, serverDict.size()); + Assert.assertEquals("TSLA", serverDict.get(3)); + Assert.assertEquals("AMZN", serverDict.get(4)); - // Third batch: no new symbols (empty delta) - accumulateDelta(serverDict, 5, new String[]{}); - Assert.assertEquals(5, serverDict.size()); + // Third batch: no new symbols (empty delta) + accumulateDelta(serverDict, 5, new String[]{}); + Assert.assertEquals(5, serverDict.size()); + }); } @Test - public void testServerSide_resolveSymbol() { - ObjList serverDict = new ObjList<>(); - serverDict.add("AAPL"); - serverDict.add("GOOG"); - serverDict.add("MSFT"); - - // Resolve by global ID - Assert.assertEquals("AAPL", serverDict.get(0)); - Assert.assertEquals("GOOG", serverDict.get(1)); - Assert.assertEquals("MSFT", serverDict.get(2)); + public void testServerSide_resolveSymbol() throws Exception { + assertMemoryLeak(() -> { + ObjList serverDict = new ObjList<>(); + serverDict.add("AAPL"); + serverDict.add("GOOG"); + serverDict.add("MSFT"); + + // Resolve by global ID + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); + }); } private void accumulateDelta(ObjList serverDict, int deltaStart, String[] symbols) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 0eeccd1..378158b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.test.AbstractTest; import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -243,25 +244,29 @@ public void testBuilderWithWebSocketTransport() { @Test public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() throws Exception { - int port; - try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { - port = s.getLocalPort(); - } - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST + ":" + port), - "connect", "Failed" - ); + assertMemoryLeak(() -> { + int port; + try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { + port = s.getLocalPort(); + } + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":" + port), + "connect", "Failed" + ); + }); } @Test public void testConnectionRefused() throws Exception { - int port = findUnusedPort(); - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST + ":" + port), - "connect", "Failed" - ); + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":" + port), + "connect", "Failed" + ); + }); } @Test @@ -283,12 +288,14 @@ public void testDisableAutoFlush_semantics() { } @Test - public void testDnsResolutionFailure() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld:9000"), - "resolve", "connect", "Failed" - ); + public void testDnsResolutionFailure() throws Exception { + assertMemoryLeak(() -> { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld:9000"), + "resolve", "connect", "Failed" + ); + }); } @Test @@ -526,7 +533,7 @@ public void testSyncModeAutoFlushDefaults() throws Exception { // Regression test: sync-mode connect() must not hardcode autoFlush to 0. // createForTesting(host, port, windowSize) mirrors what connect(h,p,tls) // creates internally. Verify it uses sensible defaults. - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1); try { Assert.assertEquals( @@ -629,25 +636,31 @@ public void testUsernamePassword_notYetSupported() { @Test public void testWsConfigString() throws Exception { - int port = findUnusedPort(); - assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); + }); } @Test public void testWsConfigString_missingAddr_fails() throws Exception { - int port = findUnusedPort(); - assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); - assertBadConfig("ws::foo=bar;", "addr is missing"); + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); + assertBadConfig("ws::foo=bar;", "addr is missing"); + }); } @Test public void testWsConfigString_protocolAlreadyConfigured_fails() throws Exception { - int port = findUnusedPort(); - assertThrowsAny( - Sender.builder("ws::addr=localhost:" + port + ";") - .enableTls(), - "TLS", "connect", "Failed" - ); + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertThrowsAny( + Sender.builder("ws::addr=localhost:" + port + ";") + .enableTls(), + "TLS", "connect", "Failed" + ); + }); } @Test @@ -668,8 +681,10 @@ public void testWsConfigString_withUsernamePassword_notYetSupported() { } @Test - public void testWssConfigString() { - assertBadConfig("wss::addr=localhost:9000;tls_verify=unsafe_off;", "connect", "Failed", "SSL"); + public void testWssConfigString() throws Exception { + assertMemoryLeak(() -> { + assertBadConfig("wss::addr=localhost:9000;tls_verify=unsafe_off;", "connect", "Failed", "SSL"); + }); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java index b5f4355..aebfc56 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -27,7 +27,7 @@ import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -45,7 +45,7 @@ public class MicrobatchBufferTest { @Test public void testAwaitRecycled() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.markSending(); @@ -74,7 +74,7 @@ public void testAwaitRecycled() throws Exception { @Test public void testAwaitRecycledWithTimeout() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.markSending(); @@ -94,7 +94,7 @@ public void testAwaitRecycledWithTimeout() throws Exception { @Test public void testBatchIdIncrementsOnReset() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { long id1 = buffer.getBatchId(); @@ -205,7 +205,7 @@ public void testConcurrentResetBatchIdUniqueness() throws Exception { @Test public void testConcurrentStateTransitions() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { AtomicReference error = new AtomicReference<>(); CountDownLatch userDone = new CountDownLatch(1); @@ -261,7 +261,7 @@ public void testConcurrentStateTransitions() throws Exception { @Test public void testConstructionWithCustomThresholds() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 100, 4096, 1_000_000_000L)) { Assert.assertEquals(1024, buffer.getBufferCapacity()); Assert.assertTrue(buffer.isFilling()); @@ -271,7 +271,7 @@ public void testConstructionWithCustomThresholds() throws Exception { @Test public void testConstructionWithDefaultThresholds() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { Assert.assertEquals(1024, buffer.getBufferCapacity()); Assert.assertEquals(0, buffer.getBufferPos()); @@ -284,7 +284,7 @@ public void testConstructionWithDefaultThresholds() throws Exception { @Test(expected = IllegalArgumentException.class) public void testConstructionWithNegativeCapacity() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer ignored = new MicrobatchBuffer(-1)) { Assert.fail("Should have thrown"); } @@ -293,7 +293,7 @@ public void testConstructionWithNegativeCapacity() throws Exception { @Test(expected = IllegalArgumentException.class) public void testConstructionWithZeroCapacity() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer ignored = new MicrobatchBuffer(0)) { Assert.fail("Should have thrown"); } @@ -302,7 +302,7 @@ public void testConstructionWithZeroCapacity() throws Exception { @Test public void testEnsureCapacityGrows() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.ensureCapacity(2000); Assert.assertTrue(buffer.getBufferCapacity() >= 2000); @@ -312,7 +312,7 @@ public void testEnsureCapacityGrows() throws Exception { @Test public void testEnsureCapacityNoGrowth() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.ensureCapacity(512); Assert.assertEquals(1024, buffer.getBufferCapacity()); // No change @@ -322,7 +322,7 @@ public void testEnsureCapacityNoGrowth() throws Exception { @Test public void testFirstRowTimeIsRecorded() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { Assert.assertEquals(0, buffer.getAgeNanos()); @@ -340,7 +340,7 @@ public void testFirstRowTimeIsRecorded() throws Exception { @Test public void testFullStateLifecycle() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { // FILLING Assert.assertTrue(buffer.isFilling()); @@ -369,7 +369,7 @@ public void testFullStateLifecycle() throws Exception { @Test public void testIncrementRowCount() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { Assert.assertEquals(0, buffer.getRowCount()); buffer.incrementRowCount(); @@ -382,7 +382,7 @@ public void testIncrementRowCount() throws Exception { @Test(expected = IllegalStateException.class) public void testIncrementRowCountWhenSealed() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.incrementRowCount(); // Should throw @@ -392,7 +392,7 @@ public void testIncrementRowCountWhenSealed() throws Exception { @Test public void testInitialState() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { Assert.assertEquals(MicrobatchBuffer.STATE_FILLING, buffer.getState()); Assert.assertTrue(buffer.isFilling()); @@ -406,7 +406,7 @@ public void testInitialState() throws Exception { @Test public void testMarkRecycledTransition() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.markSending(); @@ -421,7 +421,7 @@ public void testMarkRecycledTransition() throws Exception { @Test(expected = IllegalStateException.class) public void testMarkRecycledWhenNotSending() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.markRecycled(); // Should throw - not sending @@ -431,7 +431,7 @@ public void testMarkRecycledWhenNotSending() throws Exception { @Test public void testMarkSendingTransition() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.markSending(); @@ -445,7 +445,7 @@ public void testMarkSendingTransition() throws Exception { @Test(expected = IllegalStateException.class) public void testMarkSendingWhenNotSealed() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.markSending(); // Should throw - not sealed } @@ -454,7 +454,7 @@ public void testMarkSendingWhenNotSealed() throws Exception { @Test public void testResetFromRecycled() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.writeByte((byte) 1); buffer.incrementRowCount(); @@ -475,7 +475,7 @@ public void testResetFromRecycled() throws Exception { @Test(expected = IllegalStateException.class) public void testResetWhenSealed() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.reset(); // Should throw @@ -485,7 +485,7 @@ public void testResetWhenSealed() throws Exception { @Test(expected = IllegalStateException.class) public void testResetWhenSending() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.markSending(); @@ -496,7 +496,7 @@ public void testResetWhenSending() throws Exception { @Test public void testRollbackSealForRetry() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.writeByte((byte) 1); buffer.incrementRowCount(); @@ -518,7 +518,7 @@ public void testRollbackSealForRetry() throws Exception { @Test(expected = IllegalStateException.class) public void testRollbackSealWhenNotSealed() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.rollbackSealForRetry(); // Should throw - not sealed } @@ -527,7 +527,7 @@ public void testRollbackSealWhenNotSealed() throws Exception { @Test public void testSealTransition() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.writeByte((byte) 1); buffer.seal(); @@ -542,7 +542,7 @@ public void testSealTransition() throws Exception { @Test(expected = IllegalStateException.class) public void testSealWhenNotFilling() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.seal(); // Should throw @@ -552,7 +552,7 @@ public void testSealWhenNotFilling() throws Exception { @Test public void testSetBufferPos() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.setBufferPos(100); Assert.assertEquals(100, buffer.getBufferPos()); @@ -562,7 +562,7 @@ public void testSetBufferPos() throws Exception { @Test(expected = IllegalArgumentException.class) public void testSetBufferPosNegative() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.setBufferPos(-1); } @@ -571,7 +571,7 @@ public void testSetBufferPosNegative() throws Exception { @Test(expected = IllegalArgumentException.class) public void testSetBufferPosOutOfBounds() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.setBufferPos(2000); } @@ -580,7 +580,7 @@ public void testSetBufferPosOutOfBounds() throws Exception { @Test public void testShouldFlushAgeLimit() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { // 50ms timeout try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 0, 50_000_000L)) { buffer.writeByte((byte) 1); @@ -597,7 +597,7 @@ public void testShouldFlushAgeLimit() throws Exception { @Test public void testShouldFlushByteLimit() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 10, 0)) { for (int i = 0; i < 9; i++) { buffer.writeByte((byte) i); @@ -614,7 +614,7 @@ public void testShouldFlushByteLimit() throws Exception { @Test public void testShouldFlushEmptyBuffer() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 1, 1, 1)) { Assert.assertFalse(buffer.shouldFlush()); // Empty buffer never flushes } @@ -623,7 +623,7 @@ public void testShouldFlushEmptyBuffer() throws Exception { @Test public void testShouldFlushRowLimit() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 5, 0, 0)) { for (int i = 0; i < 4; i++) { buffer.writeByte((byte) i); @@ -640,7 +640,7 @@ public void testShouldFlushRowLimit() throws Exception { @Test public void testShouldFlushWithNoThresholds() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.writeByte((byte) 1); buffer.incrementRowCount(); @@ -660,7 +660,7 @@ public void testStateName() { @Test public void testToString() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.writeByte((byte) 1); buffer.incrementRowCount(); @@ -676,7 +676,7 @@ public void testToString() throws Exception { @Test public void testWriteBeyondInitialCapacity() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(16)) { // Write more than initial capacity for (int i = 0; i < 100; i++) { @@ -696,7 +696,7 @@ public void testWriteBeyondInitialCapacity() throws Exception { @Test public void testWriteByte() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.writeByte((byte) 0x42); Assert.assertEquals(1, buffer.getBufferPos()); @@ -710,7 +710,7 @@ public void testWriteByte() throws Exception { @Test public void testWriteFromNativeMemory() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { long src = Unsafe.malloc(10, MemoryTag.NATIVE_DEFAULT); try { // Fill source with test data @@ -736,7 +736,7 @@ public void testWriteFromNativeMemory() throws Exception { @Test public void testWriteMultipleBytes() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { for (int i = 0; i < 100; i++) { buffer.writeByte((byte) i); @@ -754,7 +754,7 @@ public void testWriteMultipleBytes() throws Exception { @Test(expected = IllegalStateException.class) public void testWriteWhenSealed() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { buffer.seal(); buffer.writeByte((byte) 1); // Should throw diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 5bf342d..f3ea891 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -34,421 +35,491 @@ public class NativeBufferWriterTest { @Test - public void testEnsureCapacityGrowsBuffer() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - assertEquals(16, writer.getCapacity()); - writer.ensureCapacity(32); - assertTrue(writer.getCapacity() >= 32); - } - } - - @Test - public void testGrowBuffer() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - // Write more than initial capacity - for (int i = 0; i < 100; i++) { - writer.putLong(i); + public void testEnsureCapacityGrowsBuffer() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + assertEquals(16, writer.getCapacity()); + writer.ensureCapacity(32); + assertTrue(writer.getCapacity() >= 32); } - Assert.assertEquals(800, writer.getPosition()); - // Verify data - for (int i = 0; i < 100; i++) { - Assert.assertEquals(i, Unsafe.getUnsafe().getLong(writer.getBufferPtr() + i * 8)); + }); + } + + @Test + public void testGrowBuffer() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // Write more than initial capacity + for (int i = 0; i < 100; i++) { + writer.putLong(i); + } + Assert.assertEquals(800, writer.getPosition()); + // Verify data + for (int i = 0; i < 100; i++) { + Assert.assertEquals(i, Unsafe.getUnsafe().getLong(writer.getBufferPtr() + i * 8)); + } } - } + }); } @Test - public void testMultipleWrites() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putByte((byte) 'I'); - writer.putByte((byte) 'L'); - writer.putByte((byte) 'P'); - writer.putByte((byte) '4'); - writer.putByte((byte) 1); // Version - writer.putByte((byte) 0); // Flags - writer.putShort((short) 1); // Table count - writer.putInt(0); // Payload length placeholder + public void testMultipleWrites() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 'I'); + writer.putByte((byte) 'L'); + writer.putByte((byte) 'P'); + writer.putByte((byte) '4'); + writer.putByte((byte) 1); // Version + writer.putByte((byte) 0); // Flags + writer.putShort((short) 1); // Table count + writer.putInt(0); // Payload length placeholder - Assert.assertEquals(12, writer.getPosition()); + Assert.assertEquals(12, writer.getPosition()); - // Verify ILP4 header - Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); - Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); - } + // Verify ILP4 header + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + } + }); } @Test - public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { - // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 - assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); - // Lone high surrogate at end: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); - // Lone low surrogate: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); - // Valid pair still works: 4 bytes - assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + }); } @Test - public void testPatchInt() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putInt(0); // Placeholder at offset 0 - writer.putInt(100); // At offset 4 - writer.patchInt(0, 42); // Patch first int - Assert.assertEquals(42, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); - Assert.assertEquals(100, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); - } + public void testPatchInt() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0); // Placeholder at offset 0 + writer.putInt(100); // At offset 4 + writer.patchInt(0, 42); // Patch first int + Assert.assertEquals(42, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + Assert.assertEquals(100, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); } @Test - public void testPatchIntAtLastValidOffset() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - writer.putLong(0L); // 8 bytes, position = 8 - // Patch at offset 4 covers bytes [4..7], exactly at the boundary - writer.patchInt(4, 0x1234); - assertEquals(0x1234, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); - } + public void testPatchIntAtLastValidOffset() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.putLong(0L); // 8 bytes, position = 8 + // Patch at offset 4 covers bytes [4..7], exactly at the boundary + writer.patchInt(4, 0x1234); + assertEquals(0x1234, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); } @Test - public void testPatchIntAtValidOffset() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - writer.putInt(0); // placeholder at offset 0 - writer.putInt(0xBEEF); // data at offset 4 - // Patch the placeholder - writer.patchInt(0, 0xCAFE); - assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); - assertEquals(0xBEEF, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); - } + public void testPatchIntAtValidOffset() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.putInt(0); // placeholder at offset 0 + writer.putInt(0xBEEF); // data at offset 4 + // Patch the placeholder + writer.patchInt(0, 0xCAFE); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + assertEquals(0xBEEF, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); + } + + @Test + public void testPutBlockOfBytes() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(); + NativeBufferWriter source = new NativeBufferWriter()) { + // Prepare source data + source.putByte((byte) 1); + source.putByte((byte) 2); + source.putByte((byte) 3); + source.putByte((byte) 4); + + // Copy to writer + writer.putBlockOfBytes(source.getBufferPtr(), 4); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals((byte) 1, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 2, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 3, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 4, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + } + }); } @Test - public void testPutBlockOfBytes() { - try (NativeBufferWriter writer = new NativeBufferWriter(); - NativeBufferWriter source = new NativeBufferWriter()) { - // Prepare source data - source.putByte((byte) 1); - source.putByte((byte) 2); - source.putByte((byte) 3); - source.putByte((byte) 4); - - // Copy to writer - writer.putBlockOfBytes(source.getBufferPtr(), 4); - Assert.assertEquals(4, writer.getPosition()); - Assert.assertEquals((byte) 1, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 2, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - Assert.assertEquals((byte) 3, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); - Assert.assertEquals((byte) 4, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); - } + public void testPutBlockOfBytesRejectsLenExceedingIntMax() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + try { + writer.putBlockOfBytes(0, (long) Integer.MAX_VALUE + 1); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("len")); + } + } + }); } @Test - public void testPutBlockOfBytesRejectsLenExceedingIntMax() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - try { - writer.putBlockOfBytes(0, (long) Integer.MAX_VALUE + 1); - fail("expected IllegalArgumentException"); - } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains("len")); + public void testPutUtf8InvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + writer.putUtf8("\uD800X"); + assertEquals(2, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); } - } + }); } @Test - public void testPutUtf8InvalidSurrogatePair() { - try (NativeBufferWriter writer = new NativeBufferWriter(64)) { - // High surrogate \uD800 followed by non-low-surrogate 'X'. - // Should produce '?' for the lone high surrogate, then 'X'. - writer.putUtf8("\uD800X"); - assertEquals(2, writer.getPosition()); - assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - } + public void testPutUtf8LoneHighSurrogateAtEnd() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uD800"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); } @Test - public void testPutUtf8LoneHighSurrogateAtEnd() { - try (NativeBufferWriter writer = new NativeBufferWriter(64)) { - writer.putUtf8("\uD800"); - assertEquals(1, writer.getPosition()); - assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - } + public void testPutUtf8LoneLowSurrogate() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uDC00"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); + } + + @Test + public void testPutUtf8LoneSurrogateMatchesUtf8Length() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // Verify putUtf8 and utf8Length agree for all lone surrogate cases + String[] cases = {"\uD800", "\uDBFF", "\uDC00", "\uDFFF", "\uD800X", "A\uDC00B"}; + for (String s : cases) { + writer.reset(); + writer.putUtf8(s); + assertEquals("length mismatch for: " + s.codePoints() + .mapToObj(cp -> String.format("U+%04X", cp)) + .reduce((a, b) -> a + " " + b).orElse(""), + NativeBufferWriter.utf8Length(s), writer.getPosition()); + } + } + }); } @Test - public void testPutUtf8LoneLowSurrogate() { - try (NativeBufferWriter writer = new NativeBufferWriter(64)) { - writer.putUtf8("\uDC00"); - assertEquals(1, writer.getPosition()); - assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - } + public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + }); } @Test - public void testPutUtf8LoneSurrogateMatchesUtf8Length() { - try (NativeBufferWriter writer = new NativeBufferWriter(64)) { - // Verify putUtf8 and utf8Length agree for all lone surrogate cases - String[] cases = {"\uD800", "\uDBFF", "\uDC00", "\uDFFF", "\uD800X", "A\uDC00B"}; - for (String s : cases) { + public void testReset() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(12345); + Assert.assertEquals(4, writer.getPosition()); writer.reset(); - writer.putUtf8(s); - assertEquals("length mismatch for: " + s.codePoints() - .mapToObj(cp -> String.format("U+%04X", cp)) - .reduce((a, b) -> a + " " + b).orElse(""), - NativeBufferWriter.utf8Length(s), writer.getPosition()); + Assert.assertEquals(0, writer.getPosition()); + // Can write again + writer.putByte((byte) 0xFF); + Assert.assertEquals(1, writer.getPosition()); } - } - } - - @Test - public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { - // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 - assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); - // Lone high surrogate at end: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); - // Lone low surrogate: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); - // Valid pair still works: 4 bytes - assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + }); } @Test - public void testReset() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putInt(12345); - Assert.assertEquals(4, writer.getPosition()); - writer.reset(); - Assert.assertEquals(0, writer.getPosition()); - // Can write again - writer.putByte((byte) 0xFF); - Assert.assertEquals(1, writer.getPosition()); - } - } - - @Test - public void testSkipAdvancesPosition() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - writer.skip(4); - assertEquals(4, writer.getPosition()); - writer.skip(8); - assertEquals(12, writer.getPosition()); - } - } - - @Test - public void testSkipBeyondCapacityGrowsBuffer() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - // skip past the 16-byte buffer — must grow, not corrupt memory - writer.skip(32); - assertEquals(32, writer.getPosition()); - assertTrue(writer.getCapacity() >= 32); - // writing after the skip must also succeed - writer.putInt(0xCAFE); - assertEquals(36, writer.getPosition()); - assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); - } + public void testSkipAdvancesPosition() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.skip(4); + assertEquals(4, writer.getPosition()); + writer.skip(8); + assertEquals(12, writer.getPosition()); + } + }); } @Test - public void testSkipThenPatchInt() { - try (NativeBufferWriter writer = new NativeBufferWriter(8)) { - int patchOffset = writer.getPosition(); - writer.skip(4); // reserve space for a length field - writer.putInt(0xDEAD); - // Patch the reserved space - writer.patchInt(patchOffset, 4); - assertEquals(0x4, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + patchOffset)); - assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); - } + public void testSkipBeyondCapacityGrowsBuffer() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // skip past the 16-byte buffer — must grow, not corrupt memory + writer.skip(32); + assertEquals(32, writer.getPosition()); + assertTrue(writer.getCapacity() >= 32); + // writing after the skip must also succeed + writer.putInt(0xCAFE); + assertEquals(36, writer.getPosition()); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + } + }); } @Test - public void testUtf8Length() { - Assert.assertEquals(0, NativeBufferWriter.utf8Length(null)); - Assert.assertEquals(0, NativeBufferWriter.utf8Length("")); - Assert.assertEquals(5, NativeBufferWriter.utf8Length("hello")); - Assert.assertEquals(2, NativeBufferWriter.utf8Length("ñ")); - Assert.assertEquals(3, NativeBufferWriter.utf8Length("€")); + public void testSkipThenPatchInt() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(8)) { + int patchOffset = writer.getPosition(); + writer.skip(4); // reserve space for a length field + writer.putInt(0xDEAD); + // Patch the reserved space + writer.patchInt(patchOffset, 4); + assertEquals(0x4, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + patchOffset)); + assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + }); } @Test - public void testWriteByte() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putByte((byte) 0x42); - Assert.assertEquals(1, writer.getPosition()); - Assert.assertEquals((byte) 0x42, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - } + public void testUtf8Length() throws Exception { + assertMemoryLeak(() -> { + Assert.assertEquals(0, NativeBufferWriter.utf8Length(null)); + Assert.assertEquals(0, NativeBufferWriter.utf8Length("")); + Assert.assertEquals(5, NativeBufferWriter.utf8Length("hello")); + Assert.assertEquals(2, NativeBufferWriter.utf8Length("ñ")); + Assert.assertEquals(3, NativeBufferWriter.utf8Length("€")); + }); } @Test - public void testWriteDouble() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putDouble(3.14159265359); - Assert.assertEquals(8, writer.getPosition()); - Assert.assertEquals(3.14159265359, Unsafe.getUnsafe().getDouble(writer.getBufferPtr()), 0.0000000001); - } + public void testWriteByte() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 0x42); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0x42, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); } @Test - public void testWriteEmptyString() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putString(""); - Assert.assertEquals(1, writer.getPosition()); - Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - } + public void testWriteDouble() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putDouble(3.14159265359); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(3.14159265359, Unsafe.getUnsafe().getDouble(writer.getBufferPtr()), 0.0000000001); + } + }); } @Test - public void testWriteFloat() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putFloat(3.14f); - Assert.assertEquals(4, writer.getPosition()); - Assert.assertEquals(3.14f, Unsafe.getUnsafe().getFloat(writer.getBufferPtr()), 0.0001f); - } + public void testWriteEmptyString() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(""); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); } @Test - public void testWriteInt() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putInt(0x12345678); - Assert.assertEquals(4, writer.getPosition()); - // Little-endian - Assert.assertEquals(0x12345678, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); - } + public void testWriteFloat() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putFloat(3.14f); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals(3.14f, Unsafe.getUnsafe().getFloat(writer.getBufferPtr()), 0.0001f); + } + }); } @Test - public void testWriteLong() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putLong(0x123456789ABCDEF0L); - Assert.assertEquals(8, writer.getPosition()); - Assert.assertEquals(0x123456789ABCDEF0L, Unsafe.getUnsafe().getLong(writer.getBufferPtr())); - } + public void testWriteInt() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0x12345678); + Assert.assertEquals(4, writer.getPosition()); + // Little-endian + Assert.assertEquals(0x12345678, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + } + }); } @Test - public void testWriteLongBigEndian() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putLongBE(0x0102030405060708L); - Assert.assertEquals(8, writer.getPosition()); - // Check big-endian byte order - long ptr = writer.getBufferPtr(); - Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(ptr)); - Assert.assertEquals((byte) 0x02, Unsafe.getUnsafe().getByte(ptr + 1)); - Assert.assertEquals((byte) 0x03, Unsafe.getUnsafe().getByte(ptr + 2)); - Assert.assertEquals((byte) 0x04, Unsafe.getUnsafe().getByte(ptr + 3)); - Assert.assertEquals((byte) 0x05, Unsafe.getUnsafe().getByte(ptr + 4)); - Assert.assertEquals((byte) 0x06, Unsafe.getUnsafe().getByte(ptr + 5)); - Assert.assertEquals((byte) 0x07, Unsafe.getUnsafe().getByte(ptr + 6)); - Assert.assertEquals((byte) 0x08, Unsafe.getUnsafe().getByte(ptr + 7)); - } + public void testWriteLong() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLong(0x123456789ABCDEF0L); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(0x123456789ABCDEF0L, Unsafe.getUnsafe().getLong(writer.getBufferPtr())); + } + }); + } + + @Test + public void testWriteLongBigEndian() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLongBE(0x0102030405060708L); + Assert.assertEquals(8, writer.getPosition()); + // Check big-endian byte order + long ptr = writer.getBufferPtr(); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 0x02, Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 0x03, Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) 0x04, Unsafe.getUnsafe().getByte(ptr + 3)); + Assert.assertEquals((byte) 0x05, Unsafe.getUnsafe().getByte(ptr + 4)); + Assert.assertEquals((byte) 0x06, Unsafe.getUnsafe().getByte(ptr + 5)); + Assert.assertEquals((byte) 0x07, Unsafe.getUnsafe().getByte(ptr + 6)); + Assert.assertEquals((byte) 0x08, Unsafe.getUnsafe().getByte(ptr + 7)); + } + }); } @Test - public void testWriteNullString() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putString(null); - Assert.assertEquals(1, writer.getPosition()); - Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - } + public void testWriteNullString() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(null); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); } @Test - public void testWriteShort() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putShort((short) 0x1234); - Assert.assertEquals(2, writer.getPosition()); - // Little-endian - Assert.assertEquals((byte) 0x34, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 0x12, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - } + public void testWriteShort() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putShort((short) 0x1234); + Assert.assertEquals(2, writer.getPosition()); + // Little-endian + Assert.assertEquals((byte) 0x34, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x12, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + }); + } + + @Test + public void testWriteString() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString("hello"); + // Length (1 byte varint) + 5 bytes + Assert.assertEquals(6, writer.getPosition()); + // Check length + Assert.assertEquals((byte) 5, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + // Check content + Assert.assertEquals((byte) 'h', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'e', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 4)); + Assert.assertEquals((byte) 'o', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 5)); + } + }); } @Test - public void testWriteString() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putString("hello"); - // Length (1 byte varint) + 5 bytes - Assert.assertEquals(6, writer.getPosition()); - // Check length - Assert.assertEquals((byte) 5, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - // Check content - Assert.assertEquals((byte) 'h', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - Assert.assertEquals((byte) 'e', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); - Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); - Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 4)); - Assert.assertEquals((byte) 'o', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 5)); - } + public void testWriteUtf8Ascii() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putUtf8("ABC"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 'A', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'B', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'C', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + }); } @Test - public void testWriteUtf8Ascii() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putUtf8("ABC"); - Assert.assertEquals(3, writer.getPosition()); - Assert.assertEquals((byte) 'A', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 'B', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - Assert.assertEquals((byte) 'C', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); - } + public void testWriteUtf8ThreeByte() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // € is 3 bytes in UTF-8 + writer.putUtf8("€"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 0xE2, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x82, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0xAC, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + }); } @Test - public void testWriteUtf8ThreeByte() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - // € is 3 bytes in UTF-8 - writer.putUtf8("€"); - Assert.assertEquals(3, writer.getPosition()); - Assert.assertEquals((byte) 0xE2, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 0x82, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - Assert.assertEquals((byte) 0xAC, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); - } + public void testWriteUtf8TwoByte() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // ñ is 2 bytes in UTF-8 + writer.putUtf8("ñ"); + Assert.assertEquals(2, writer.getPosition()); + Assert.assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0xB1, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + }); } @Test - public void testWriteUtf8TwoByte() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - // ñ is 2 bytes in UTF-8 - writer.putUtf8("ñ"); - Assert.assertEquals(2, writer.getPosition()); - Assert.assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 0xB1, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - } + public void testWriteVarintLarge() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Test larger value + writer.putVarint(16384); + Assert.assertEquals(3, writer.getPosition()); + // LEB128: 16384 = 0x80 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + }); } @Test - public void testWriteVarintLarge() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - // Test larger value - writer.putVarint(16384); - Assert.assertEquals(3, writer.getPosition()); - // LEB128: 16384 = 0x80 0x80 0x01 - Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); - } - } - - @Test - public void testWriteVarintMedium() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - // Two bytes for 128 - writer.putVarint(128); - Assert.assertEquals(2, writer.getPosition()); - // LEB128: 128 = 0x80 0x01 - Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); - } + public void testWriteVarintMedium() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Two bytes for 128 + writer.putVarint(128); + Assert.assertEquals(2, writer.getPosition()); + // LEB128: 128 = 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + }); } @Test - public void testWriteVarintSmall() { - try (NativeBufferWriter writer = new NativeBufferWriter()) { - // Single byte for values < 128 - writer.putVarint(127); - Assert.assertEquals(1, writer.getPosition()); - Assert.assertEquals((byte) 127, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - } + public void testWriteVarintSmall() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Single byte for values < 128 + writer.putVarint(127); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 127, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + }); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java index b26f488..1fa657e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java @@ -27,7 +27,7 @@ import io.questdb.client.cutlass.qwp.client.InFlightWindow; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.test.AbstractTest; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -43,7 +43,7 @@ public class QwpDeltaDictRollbackTest extends AbstractTest { @Test public void testSyncFlushFailureDoesNotAdvanceMaxSentSymbolId() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { // Sync mode (window=1), not connected to any server QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 5a298e9..d5909c3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -40,1215 +41,1322 @@ public class QwpWebSocketEncoderTest { @Test - public void testBufferResetAndReuse() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test"); - - // First batch - for (int i = 0; i < 100; i++) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(i); - buffer.nextRow(); - } - int size1 = encoder.encode(buffer, false); - - // Reset and second batch - buffer.reset(); - for (int i = 0; i < 50; i++) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(i * 2); - buffer.nextRow(); + public void testBufferResetAndReuse() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + // First batch + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + int size1 = encoder.encode(buffer, false); + + // Reset and second batch + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i * 2); + buffer.nextRow(); + } + int size2 = encoder.encode(buffer, false); + + Assert.assertTrue(size1 > size2); // More rows = larger + Assert.assertEquals(50, buffer.getRowCount()); } - int size2 = encoder.encode(buffer, false); - - Assert.assertTrue(size1 > size2); // More rows = larger - Assert.assertEquals(50, buffer.getRowCount()); - } + }); } @Test - public void testEncode2DDoubleArray() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncode2DDoubleArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncode2DLongArray() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncode2DLongArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_LONG_ARRAY, true); - col.addLongArray(new long[][]{{1L, 2L}, {3L, 4L}}); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[][]{{1L, 2L}, {3L, 4L}}); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncode3DDoubleArray() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("tensor", TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(new double[][][]{ - {{1.0, 2.0}, {3.0, 4.0}}, - {{5.0, 6.0}, {7.0, 8.0}} - }); - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } - } + public void testEncode3DDoubleArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("tensor", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][][]{ + {{1.0, 2.0}, {3.0, 4.0}}, + {{5.0, 6.0}, {7.0, 8.0}} + }); + buffer.nextRow(); - @Test - public void testEncodeAllBasicTypesInOneRow() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("all_types"); - - buffer.getOrCreateColumn("b", TYPE_BOOLEAN, false).addBoolean(true); - buffer.getOrCreateColumn("by", TYPE_BYTE, false).addByte((byte) 42); - buffer.getOrCreateColumn("sh", TYPE_SHORT, false).addShort((short) 1000); - buffer.getOrCreateColumn("i", TYPE_INT, false).addInt(100000); - buffer.getOrCreateColumn("l", TYPE_LONG, false).addLong(1000000000L); - buffer.getOrCreateColumn("f", TYPE_FLOAT, false).addFloat(3.14f); - buffer.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(3.14159265); - buffer.getOrCreateColumn("s", TYPE_STRING, true).addString("test"); - buffer.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbol("AAPL"); - buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1000000L); - - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(1, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeAllBooleanValues() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeAllBasicTypesInOneRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("all_types")) { + + buffer.getOrCreateColumn("b", TYPE_BOOLEAN, false).addBoolean(true); + buffer.getOrCreateColumn("by", TYPE_BYTE, false).addByte((byte) 42); + buffer.getOrCreateColumn("sh", TYPE_SHORT, false).addShort((short) 1000); + buffer.getOrCreateColumn("i", TYPE_INT, false).addInt(100000); + buffer.getOrCreateColumn("l", TYPE_LONG, false).addLong(1000000000L); + buffer.getOrCreateColumn("f", TYPE_FLOAT, false).addFloat(3.14f); + buffer.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(3.14159265); + buffer.getOrCreateColumn("s", TYPE_STRING, true).addString("test"); + buffer.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbol("AAPL"); + buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1000000L); - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("flag", TYPE_BOOLEAN, false); - for (int i = 0; i < 100; i++) { - col.addBoolean(i % 2 == 0); buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(1, buffer.getRowCount()); } + }); + } - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(100, buffer.getRowCount()); - } + @Test + public void testEncodeAllBooleanValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("flag", TYPE_BOOLEAN, false); + for (int i = 0; i < 100; i++) { + col.addBoolean(i % 2 == 0); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + }); } @Test - public void testEncodeDecimal128() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeDecimal128() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("amount", TYPE_DECIMAL128, false); - col.addDecimal128(io.questdb.client.std.Decimal128.fromLong(123456789012345L, 4)); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("amount", TYPE_DECIMAL128, false); + col.addDecimal128(io.questdb.client.std.Decimal128.fromLong(123456789012345L, 4)); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeDecimal256() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeDecimal256() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("bignum", TYPE_DECIMAL256, false); - col.addDecimal256(io.questdb.client.std.Decimal256.fromLong(Long.MAX_VALUE, 6)); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("bignum", TYPE_DECIMAL256, false); + col.addDecimal256(io.questdb.client.std.Decimal256.fromLong(Long.MAX_VALUE, 6)); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeDecimal64() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeDecimal64() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); - col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeDoubleArray() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test"); + public void testEncodeDoubleArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(new double[]{1.0, 2.0, 3.0}); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[]{1.0, 2.0, 3.0}); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeEmptyString() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeEmptyString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); - col.addString(""); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(""); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeEmptyTableName() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - // Edge case: empty table name (probably invalid but let's verify encoding works) - QwpTableBuffer buffer = new QwpTableBuffer(""); - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(1L); - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 0); - } - } + public void testEncodeEmptyTableName() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("")) { + // Edge case: empty table name (probably invalid but let's verify encoding works) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); - @Test - public void testEncodeLargeArray() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - // Large 1D array - double[] largeArray = new double[1000]; - for (int i = 0; i < 1000; i++) { - largeArray[i] = i * 1.5; + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 0); } - - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(largeArray); - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 8000); // At least 8 bytes per double - } + }); } @Test - public void testEncodeLargeRowCount() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("metrics"); - - for (int i = 0; i < 10000; i++) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(i); + public void testEncodeLargeArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + // Large 1D array + double[] largeArray = new double[1000]; + for (int i = 0; i < 1000; i++) { + largeArray[i] = i * 1.5; + } + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(largeArray); buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 8000); // At least 8 bytes per double } + }); + } - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(10000, buffer.getRowCount()); - } + @Test + public void testEncodeLargeRowCount() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("metrics")) { + + for (int i = 0; i < 10_000; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10_000, buffer.getRowCount()); + } + }); } @Test - public void testEncodeLongArray() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test"); + public void testEncodeLongArray() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_LONG_ARRAY, true); - col.addLongArray(new long[]{1L, 2L, 3L}); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[]{1L, 2L, 3L}); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } // ==================== SYMBOL COLUMN TESTS ==================== @Test - public void testEncodeLongString() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeLongString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - String sb = "a".repeat(10000); + String sb = "a".repeat(10_000); - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("data", TYPE_STRING, true); - col.addString(sb); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("data", TYPE_STRING, true); + col.addString(sb); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 10000); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 10_000); + } + }); } @Test - public void testEncodeMaxMinLong() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeMaxMinLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(Long.MAX_VALUE); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(Long.MAX_VALUE); + buffer.nextRow(); - col.addLong(Long.MIN_VALUE); - buffer.nextRow(); + col.addLong(Long.MIN_VALUE); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(2, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); } @Test - public void testEncodeMixedColumnTypes() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("events"); + public void testEncodeMixedColumnTypes() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("events")) { - // Add columns of different types - QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); - symbolCol.addSymbol("server1"); + // Add columns of different types + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server1"); - QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); - longCol.addLong(42); + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(42); - QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); - doubleCol.addDouble(3.14); + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(3.14); - QwpTableBuffer.ColumnBuffer boolCol = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); - boolCol.addBoolean(true); + QwpTableBuffer.ColumnBuffer boolCol = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + boolCol.addBoolean(true); - QwpTableBuffer.ColumnBuffer stringCol = buffer.getOrCreateColumn("message", TYPE_STRING, true); - stringCol.addString("hello world"); + QwpTableBuffer.ColumnBuffer stringCol = buffer.getOrCreateColumn("message", TYPE_STRING, true); + stringCol.addString("hello world"); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - tsCol.addLong(1000000L); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L); - buffer.nextRow(); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeMixedColumnsMultipleRows() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("events"); + public void testEncodeMixedColumnsMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("events")) { - for (int i = 0; i < 50; i++) { - QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); - symbolCol.addSymbol("server" + (i % 5)); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server" + (i % 5)); - QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); - longCol.addLong(i * 10); + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(i * 10); - QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); - doubleCol.addDouble(i * 1.5); + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(i * 1.5); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - tsCol.addLong(1000000L + i); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L + i); - buffer.nextRow(); - } + buffer.nextRow(); + } - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(50, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(50, buffer.getRowCount()); + } + }); } // ==================== UUID COLUMN TESTS ==================== @Test - public void testEncodeMultipleColumns() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("weather"); + public void testEncodeMultipleColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("weather")) { - // Add multiple columns - QwpTableBuffer.ColumnBuffer tempCol = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); - tempCol.addDouble(23.5); + // Add multiple columns + QwpTableBuffer.ColumnBuffer tempCol = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + tempCol.addDouble(23.5); - QwpTableBuffer.ColumnBuffer humCol = buffer.getOrCreateColumn("humidity", TYPE_LONG, false); - humCol.addLong(65); + QwpTableBuffer.ColumnBuffer humCol = buffer.getOrCreateColumn("humidity", TYPE_LONG, false); + humCol.addLong(65); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - tsCol.addLong(1000000L); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L); - buffer.nextRow(); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); - // Verify header - QwpBufferWriter buf = encoder.getBuffer(); - long ptr = buf.getBufferPtr(); - Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); - Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); - Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); - Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); - } + // Verify header + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + } + }); } @Test - public void testEncodeMultipleDecimal64() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeMultipleDecimal64() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); - col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); - col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(67890L, 2)); // 678.90 - buffer.nextRow(); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(67890L, 2)); // 678.90 + buffer.nextRow(); - col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(11111L, 2)); // 111.11 - buffer.nextRow(); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(11111L, 2)); // 111.11 + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(3, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + }); } // ==================== DECIMAL COLUMN TESTS ==================== @Test - public void testEncodeMultipleRows() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("metrics"); + public void testEncodeMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("metrics")) { - for (int i = 0; i < 100; i++) { - QwpTableBuffer.ColumnBuffer valCol = buffer.getOrCreateColumn("value", TYPE_LONG, false); - valCol.addLong(i); + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer valCol = buffer.getOrCreateColumn("value", TYPE_LONG, false); + valCol.addLong(i); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - tsCol.addLong(1000000L + i); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L + i); - buffer.nextRow(); - } + buffer.nextRow(); + } - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(100, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + }); } @Test - public void testEncodeMultipleSymbolsSameDictionary() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeMultipleSymbolsSameDictionary() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); - col.addSymbol("server1"); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); - col.addSymbol("server1"); // Same symbol - buffer.nextRow(); + col.addSymbol("server1"); // Same symbol + buffer.nextRow(); - col.addSymbol("server2"); // Different symbol - buffer.nextRow(); + col.addSymbol("server2"); // Different symbol + buffer.nextRow(); - col.addSymbol("server1"); // Back to first - buffer.nextRow(); + col.addSymbol("server1"); // Back to first + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(4, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + }); } @Test - public void testEncodeMultipleUuids() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); - for (int i = 0; i < 10; i++) { - col.addUuid(i * 1000L, i * 2000L); - buffer.nextRow(); + public void testEncodeMultipleUuids() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + for (int i = 0; i < 10; i++) { + col.addUuid(i * 1000L, i * 2000L); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10, buffer.getRowCount()); } - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(10, buffer.getRowCount()); - } + }); } @Test - public void testEncodeNaNDouble() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeNaNDouble() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); - col.addDouble(Double.NaN); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.NaN); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } // ==================== ARRAY COLUMN TESTS ==================== @Test - public void testEncodeNegativeLong() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeNegativeLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(-123456789L); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(-123456789L); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeNullableColumnWithNull() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test"); - - // Nullable column with null - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); - col.addString(null); - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + public void testEncodeNullableColumnWithNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + // Nullable column with null + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(null); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeNullableColumnWithValue() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test"); - - // Nullable column with a value - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); - col.addString("hello"); - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + public void testEncodeNullableColumnWithValue() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + + // Nullable column with a value + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeNullableSymbolWithNull() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeNullableSymbolWithNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, true); - col.addSymbol("server1"); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, true); + col.addSymbol("server1"); + buffer.nextRow(); - col.addSymbol(null); // Null symbol - buffer.nextRow(); + col.addSymbol(null); // Null symbol + buffer.nextRow(); - col.addSymbol("server2"); - buffer.nextRow(); + col.addSymbol("server2"); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(3, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + }); } // ==================== MULTIPLE ROWS TESTS ==================== @Test - public void testEncodeSingleRowWithBoolean() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeSingleRowWithBoolean() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); - col.addBoolean(true); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + col.addBoolean(true); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeSingleRowWithDouble() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeSingleRowWithDouble() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); - col.addDouble(23.5); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + col.addDouble(23.5); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } // ==================== MIXED COLUMN TYPES ==================== @Test - public void testEncodeSingleRowWithLong() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - // Add a long column - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("value", TYPE_LONG, false); - col.addLong(12345L); - buffer.nextRow(); + public void testEncodeSingleRowWithLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + // Add a long column + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("value", TYPE_LONG, false); + col.addLong(12345L); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); // At least header size + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); // At least header size - QwpBufferWriter buf = encoder.getBuffer(); - long ptr = buf.getBufferPtr(); + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); - // Verify header magic - Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); - Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); - Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); - Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + // Verify header magic + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); - // Version - Assert.assertEquals(VERSION_1, Unsafe.getUnsafe().getByte(ptr + 4)); + // Version + Assert.assertEquals(VERSION_1, Unsafe.getUnsafe().getByte(ptr + 4)); - // Table count (little-endian short) - Assert.assertEquals((short) 1, Unsafe.getUnsafe().getShort(ptr + 6)); - } + // Table count (little-endian short) + Assert.assertEquals((short) 1, Unsafe.getUnsafe().getShort(ptr + 6)); + } + }); } @Test - public void testEncodeSingleRowWithString() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeSingleRowWithString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); - col.addString("hello"); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } // ==================== EDGE CASES ==================== @Test - public void testEncodeSingleRowWithTimestamp() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - // Add a timestamp column (designated timestamp uses empty name) - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - col.addLong(1000000L); // Micros - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + public void testEncodeSingleRowWithTimestamp() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + // Add a timestamp column (designated timestamp uses empty name) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); // Micros + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeSingleSymbol() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeSingleSymbol() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); - col.addSymbol("server1"); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeSpecialDoubles() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeSpecialDoubles() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); - col.addDouble(Double.MAX_VALUE); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.MAX_VALUE); + buffer.nextRow(); - col.addDouble(Double.MIN_VALUE); - buffer.nextRow(); + col.addDouble(Double.MIN_VALUE); + buffer.nextRow(); - col.addDouble(Double.POSITIVE_INFINITY); - buffer.nextRow(); + col.addDouble(Double.POSITIVE_INFINITY); + buffer.nextRow(); - col.addDouble(Double.NEGATIVE_INFINITY); - buffer.nextRow(); + col.addDouble(Double.NEGATIVE_INFINITY); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(4, buffer.getRowCount()); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + }); } @Test - public void testEncodeSymbolWithManyDistinctValues() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); - for (int i = 0; i < 100; i++) { - col.addSymbol("server" + i); - buffer.nextRow(); + public void testEncodeSymbolWithManyDistinctValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + for (int i = 0; i < 100; i++) { + col.addSymbol("server" + i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); } - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - Assert.assertEquals(100, buffer.getRowCount()); - } + }); } @Test - public void testEncodeUnicodeString() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeUnicodeString() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); - col.addString("Hello 世界 🌍"); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("Hello 世界 🌍"); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeUuid() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeUuid() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); - col.addUuid(0x123456789ABCDEF0L, 0xFEDCBA9876543210L); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + col.addUuid(0x123456789ABCDEF0L, 0xFEDCBA9876543210L); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncodeWithDeltaDict_freshConnection_sendsAllSymbols() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - // Add symbol column with global IDs - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); - - // Simulate adding symbols via global dictionary - int id1 = globalDict.getOrAddSymbol("AAPL"); // ID 0 - int id2 = globalDict.getOrAddSymbol("GOOG"); // ID 1 - col.addSymbolWithGlobalId("AAPL", id1); - buffer.nextRow(); - col.addSymbolWithGlobalId("GOOG", id2); - buffer.nextRow(); - - // Fresh connection: confirmedMaxId = -1, so delta should include all symbols (0, 1) - int confirmedMaxId = -1; - int batchMaxId = 1; - - int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); - Assert.assertTrue(size > 12); - - QwpBufferWriter buf = encoder.getBuffer(); - long ptr = buf.getBufferPtr(); - - // Verify header flag has FLAG_DELTA_SYMBOL_DICT set - byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); - Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); - } + public void testEncodeWithDeltaDict_freshConnection_sendsAllSymbols() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Add symbol column with global IDs + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + + // Simulate adding symbols via global dictionary + int id1 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id2 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + col.addSymbolWithGlobalId("AAPL", id1); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id2); + buffer.nextRow(); + + // Fresh connection: confirmedMaxId = -1, so delta should include all symbols (0, 1) + int confirmedMaxId = -1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify header flag has FLAG_DELTA_SYMBOL_DICT set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + }); } @Test - public void testEncodeWithDeltaDict_noNewSymbols_sendsEmptyDelta() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - // Pre-populate dictionary with all symbols - int id0 = globalDict.getOrAddSymbol("AAPL"); // ID 0 - int id1 = globalDict.getOrAddSymbol("GOOG"); // ID 1 - - // Use only existing symbols - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); - col.addSymbolWithGlobalId("AAPL", id0); - buffer.nextRow(); - col.addSymbolWithGlobalId("GOOG", id1); - buffer.nextRow(); - - // Server has confirmed all symbols (0-1), batchMaxId is 1 - int confirmedMaxId = 1; - int batchMaxId = 1; - - int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); - Assert.assertTrue(size > 12); - - QwpBufferWriter buf = encoder.getBuffer(); - long ptr = buf.getBufferPtr(); - - // Verify delta flag is set - byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); - Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); - - // Read delta section after header - long pos = ptr + HEADER_SIZE; - - // Read deltaStart varint (should be 2 = confirmedMaxId + 1) - int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; - Assert.assertEquals(2, deltaStart); - pos++; - - // Read deltaCount varint (should be 0) - int deltaCount = Unsafe.getUnsafe().getByte(pos) & 0x7F; - Assert.assertEquals(0, deltaCount); - } + public void testEncodeWithDeltaDict_noNewSymbols_sendsEmptyDelta() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Pre-populate dictionary with all symbols + int id0 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id1 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Use only existing symbols + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", id0); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id1); + buffer.nextRow(); + + // Server has confirmed all symbols (0-1), batchMaxId is 1 + int confirmedMaxId = 1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + pos++; + + // Read deltaCount varint (should be 0) + int deltaCount = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(0, deltaCount); + } + }); } @Test - public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - - // Pre-populate dictionary (simulating symbols already sent) - globalDict.getOrAddSymbol("AAPL"); // ID 0 - globalDict.getOrAddSymbol("GOOG"); // ID 1 - - // Now add new symbols - int id2 = globalDict.getOrAddSymbol("MSFT"); // ID 2 - int id3 = globalDict.getOrAddSymbol("TSLA"); // ID 3 - - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); - col.addSymbolWithGlobalId("MSFT", id2); - buffer.nextRow(); - col.addSymbolWithGlobalId("TSLA", id3); - buffer.nextRow(); - - // Server has confirmed IDs 0-1, so delta should only include 2-3 - int confirmedMaxId = 1; - int batchMaxId = 3; - - int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); - Assert.assertTrue(size > 12); - - QwpBufferWriter buf = encoder.getBuffer(); - long ptr = buf.getBufferPtr(); - - // Verify delta flag is set - byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); - Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); - - // Read delta section after header - long pos = ptr + HEADER_SIZE; - - // Read deltaStart varint (should be 2 = confirmedMaxId + 1) - int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; - Assert.assertEquals(2, deltaStart); - } + public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Pre-populate dictionary (simulating symbols already sent) + globalDict.getOrAddSymbol("AAPL"); // ID 0 + globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Now add new symbols + int id2 = globalDict.getOrAddSymbol("MSFT"); // ID 2 + int id3 = globalDict.getOrAddSymbol("TSLA"); // ID 3 + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("MSFT", id2); + buffer.nextRow(); + col.addSymbolWithGlobalId("TSLA", id3); + buffer.nextRow(); + + // Server has confirmed IDs 0-1, so delta should only include 2-3 + int confirmedMaxId = 1; + int batchMaxId = 3; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + } + }); } // ==================== SCHEMA REFERENCE TESTS ==================== @Test - public void testEncodeWithSchemaRef() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeWithSchemaRef() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(42L); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); - int size = encoder.encode(buffer, true); // Use schema reference - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, true); // Use schema reference + Assert.assertTrue(size > 12); + } + }); } // ==================== BUFFER REUSE TESTS ==================== @Test - public void testEncodeZeroLong() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + public void testEncodeZeroLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(0L); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(0L); + buffer.nextRow(); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testEncoderReusability() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - // Encode first message - QwpTableBuffer buffer1 = new QwpTableBuffer("table1"); - QwpTableBuffer.ColumnBuffer col1 = buffer1.getOrCreateColumn("x", TYPE_LONG, false); - col1.addLong(1L); - buffer1.nextRow(); - int size1 = encoder.encode(buffer1, false); - - // Encode second message (encoder should reset internally) - QwpTableBuffer buffer2 = new QwpTableBuffer("table2"); - QwpTableBuffer.ColumnBuffer col2 = buffer2.getOrCreateColumn("y", TYPE_DOUBLE, false); - col2.addDouble(2.0); - buffer2.nextRow(); - int size2 = encoder.encode(buffer2, false); - - // Both should succeed - Assert.assertTrue(size1 > 12); - Assert.assertTrue(size2 > 12); - } + public void testEncoderReusability() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer1 = new QwpTableBuffer("table1"); + QwpTableBuffer buffer2 = new QwpTableBuffer("table2")) { + // Encode first message + QwpTableBuffer.ColumnBuffer col1 = buffer1.getOrCreateColumn("x", TYPE_LONG, false); + col1.addLong(1L); + buffer1.nextRow(); + int size1 = encoder.encode(buffer1, false); + + // Encode second message (encoder should reset internally) + QwpTableBuffer.ColumnBuffer col2 = buffer2.getOrCreateColumn("y", TYPE_DOUBLE, false); + col2.addDouble(2.0); + buffer2.nextRow(); + int size2 = encoder.encode(buffer2, false); + + // Both should succeed + Assert.assertTrue(size1 > 12); + Assert.assertTrue(size2 > 12); + } + }); } // ==================== ALL BASIC TYPES IN ONE ROW ==================== @Test - public void testGlobalSymbolDictionaryBasics() { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - - // Test sequential IDs - Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); - Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); - Assert.assertEquals(2, dict.getOrAddSymbol("MSFT")); - - // Test deduplication - Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); - Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); - - // Test retrieval - Assert.assertEquals("AAPL", dict.getSymbol(0)); - Assert.assertEquals("GOOG", dict.getSymbol(1)); - Assert.assertEquals("MSFT", dict.getSymbol(2)); - - // Test size - Assert.assertEquals(3, dict.size()); + public void testGlobalSymbolDictionaryBasics() throws Exception { + assertMemoryLeak(() -> { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Test sequential IDs + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + Assert.assertEquals(2, dict.getOrAddSymbol("MSFT")); + + // Test deduplication + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + + // Test retrieval + Assert.assertEquals("AAPL", dict.getSymbol(0)); + Assert.assertEquals("GOOG", dict.getSymbol(1)); + Assert.assertEquals("MSFT", dict.getSymbol(2)); + + // Test size + Assert.assertEquals(3, dict.size()); + }); } // ==================== Delta Symbol Dictionary Tests ==================== @Test - public void testGorillaEncoding_compressionRatio() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); - - QwpTableBuffer buffer = new QwpTableBuffer("metrics"); - - // Add many timestamps with constant delta - best case for Gorilla - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); - for (int i = 0; i < 1000; i++) { - col.addLong(1000000000L + i * 1000L); - buffer.nextRow(); + public void testGorillaEncoding_compressionRatio() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("metrics")) { + encoder.setGorillaEnabled(true); + + // Add many timestamps with constant delta - best case for Gorilla + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Calculate theoretical minimum size for Gorilla: + // - Header: 12 bytes + // - Table header, column schema, etc. + // - First timestamp: 8 bytes + // - Second timestamp: 8 bytes + // - Remaining 998 timestamps: 998 bits (1 bit each for DoD=0) = ~125 bytes + + // Calculate size without Gorilla (1000 * 8 = 8000 bytes just for timestamps) + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // For constant delta, Gorilla should achieve significant compression + double compressionRatio = (double) sizeWithGorilla / sizeWithoutGorilla; + Assert.assertTrue("Compression ratio should be < 0.2 for constant delta", + compressionRatio < 0.2); } - - int sizeWithGorilla = encoder.encode(buffer, false); - - // Calculate theoretical minimum size for Gorilla: - // - Header: 12 bytes - // - Table header, column schema, etc. - // - First timestamp: 8 bytes - // - Second timestamp: 8 bytes - // - Remaining 998 timestamps: 998 bits (1 bit each for DoD=0) = ~125 bytes - - // Calculate size without Gorilla (1000 * 8 = 8000 bytes just for timestamps) - encoder.setGorillaEnabled(false); - buffer.reset(); - col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); - for (int i = 0; i < 1000; i++) { - col.addLong(1000000000L + i * 1000L); - buffer.nextRow(); - } - - int sizeWithoutGorilla = encoder.encode(buffer, false); - - // For constant delta, Gorilla should achieve significant compression - double compressionRatio = (double) sizeWithGorilla / sizeWithoutGorilla; - Assert.assertTrue("Compression ratio should be < 0.2 for constant delta", - compressionRatio < 0.2); - } + }); } @Test - public void testGorillaEncoding_multipleTimestampColumns() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); + public void testGorillaEncoding_multipleTimestampColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); - QwpTableBuffer buffer = new QwpTableBuffer("test"); + // Add multiple timestamp columns + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); - // Add multiple timestamp columns - for (int i = 0; i < 50; i++) { - QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); - ts1Col.addLong(1000000000L + i * 1000L); + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); - QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); - ts2Col.addLong(2000000000L + i * 2000L); - - buffer.nextRow(); - } + buffer.nextRow(); + } - int sizeWithGorilla = encoder.encode(buffer, false); + int sizeWithGorilla = encoder.encode(buffer, false); - // Compare with uncompressed - encoder.setGorillaEnabled(false); - buffer.reset(); - for (int i = 0; i < 50; i++) { - QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); - ts1Col.addLong(1000000000L + i * 1000L); + // Compare with uncompressed + encoder.setGorillaEnabled(false); + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); - QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); - ts2Col.addLong(2000000000L + i * 2000L); + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); - buffer.nextRow(); - } + buffer.nextRow(); + } - int sizeWithoutGorilla = encoder.encode(buffer, false); + int sizeWithoutGorilla = encoder.encode(buffer, false); - Assert.assertTrue("Gorilla should compress multiple timestamp columns", - sizeWithGorilla < sizeWithoutGorilla); - } + Assert.assertTrue("Gorilla should compress multiple timestamp columns", + sizeWithGorilla < sizeWithoutGorilla); + } + }); } @Test - public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); - - QwpTableBuffer buffer = new QwpTableBuffer("test"); - - // Add multiple timestamps with constant delta (best compression) - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - for (int i = 0; i < 100; i++) { - col.addLong(1000000000L + i * 1000L); - buffer.nextRow(); + public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Add multiple timestamps with constant delta (best compression) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Now encode without Gorilla + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // Gorilla should produce smaller output for constant-delta timestamps + Assert.assertTrue("Gorilla encoding should be smaller", + sizeWithGorilla < sizeWithoutGorilla); } + }); + } - int sizeWithGorilla = encoder.encode(buffer, false); - - // Now encode without Gorilla - encoder.setGorillaEnabled(false); - buffer.reset(); - col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - for (int i = 0; i < 100; i++) { - col.addLong(1000000000L + i * 1000L); - buffer.nextRow(); + @Test + public void testGorillaEncoding_nanosTimestamps() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Use TYPE_TIMESTAMP_NANOS + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP_NANOS, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000000000000L + i * 1000000L); // Nanos with millisecond intervals + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); } - - int sizeWithoutGorilla = encoder.encode(buffer, false); - - // Gorilla should produce smaller output for constant-delta timestamps - Assert.assertTrue("Gorilla encoding should be smaller", - sizeWithGorilla < sizeWithoutGorilla); - } + }); } @Test - public void testGorillaEncoding_nanosTimestamps() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); - - QwpTableBuffer buffer = new QwpTableBuffer("test"); - - // Use TYPE_TIMESTAMP_NANOS - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP_NANOS, true); - for (int i = 0; i < 100; i++) { - col.addLong(1000000000000000000L + i * 1000000L); // Nanos with millisecond intervals + public void testGorillaEncoding_singleTimestamp_usesUncompressed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Single timestamp - should use uncompressed + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); buffer.nextRow(); - } - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - - // Verify header has Gorilla flag - QwpBufferWriter buf = encoder.getBuffer(); - byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); - Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); - } + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + }); } @Test - public void testGorillaEncoding_singleTimestamp_usesUncompressed() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); - - QwpTableBuffer buffer = new QwpTableBuffer("test"); + public void testGorillaEncoding_twoTimestamps_usesUncompressed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Only 2 timestamps - should use uncompressed (Gorilla needs 3+) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + col.addLong(2000000L); + buffer.nextRow(); - // Single timestamp - should use uncompressed - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - col.addLong(1000000L); - buffer.nextRow(); + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - } + // Verify header has Gorilla flag set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + }); } @Test - public void testGorillaEncoding_twoTimestamps_usesUncompressed() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); - - QwpTableBuffer buffer = new QwpTableBuffer("test"); - - // Only 2 timestamps - should use uncompressed (Gorilla needs 3+) - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - col.addLong(1000000L); - buffer.nextRow(); - col.addLong(2000000L); - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); - - // Verify header has Gorilla flag set - QwpBufferWriter buf = encoder.getBuffer(); - byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); - Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); - } + public void testGorillaEncoding_varyingDelta() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + + // Varying deltas that exercise different buckets + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + long[] timestamps = { + 1000000000L, + 1000001000L, // delta=1000 + 1000002000L, // DoD=0 + 1000003050L, // DoD=50 + 1000004200L, // DoD=100 + 1000006200L, // DoD=850 + }; + + for (long ts : timestamps) { + col.addLong(ts); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + }); } @Test - public void testGorillaEncoding_varyingDelta() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); - - QwpTableBuffer buffer = new QwpTableBuffer("test"); - - // Varying deltas that exercise different buckets - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); - long[] timestamps = { - 1000000000L, - 1000001000L, // delta=1000 - 1000002000L, // DoD=0 - 1000003050L, // DoD=50 - 1000004200L, // DoD=100 - 1000006200L, // DoD=850 - }; - - for (long ts : timestamps) { - col.addLong(ts); + public void testGorillaFlagDisabled() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(false); + Assert.assertFalse(encoder.isGorillaEnabled()); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); buffer.nextRow(); - } - int size = encoder.encode(buffer, false); - Assert.assertTrue(size > 12); + encoder.encode(buffer, false); - // Verify header has Gorilla flag - QwpBufferWriter buf = encoder.getBuffer(); - byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); - Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); - } + // Check flags byte doesn't have Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(0, flags & FLAG_GORILLA); + } + }); } @Test - public void testGorillaFlagDisabled() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(false); - Assert.assertFalse(encoder.isGorillaEnabled()); - - QwpTableBuffer buffer = new QwpTableBuffer("test"); - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); - col.addLong(1000000L); - buffer.nextRow(); - - encoder.encode(buffer, false); - - // Check flags byte doesn't have Gorilla bit set - QwpBufferWriter buf = encoder.getBuffer(); - byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); - Assert.assertEquals(0, flags & FLAG_GORILLA); - } - } + public void testGorillaFlagEnabled() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { + encoder.setGorillaEnabled(true); + Assert.assertTrue(encoder.isGorillaEnabled()); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); - @Test - public void testGorillaFlagEnabled() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(true); - Assert.assertTrue(encoder.isGorillaEnabled()); - - QwpTableBuffer buffer = new QwpTableBuffer("test"); - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); - col.addLong(1000000L); - buffer.nextRow(); - - encoder.encode(buffer, false); - - // Check flags byte has Gorilla bit set - QwpBufferWriter buf = encoder.getBuffer(); - byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); - Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); - } + encoder.encode(buffer, false); + + // Check flags byte has Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + }); } @Test - public void testPayloadLengthPatched() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test_table"); - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(42L); - buffer.nextRow(); - - int size = encoder.encode(buffer, false); - - // Payload length is at offset 8 (4 magic + 1 version + 1 flags + 2 tablecount) - QwpBufferWriter buf = encoder.getBuffer(); - int payloadLength = Unsafe.getUnsafe().getInt(buf.getBufferPtr() + 8); - - // Payload length should be total size minus header (12 bytes) - Assert.assertEquals(size - 12, payloadLength); - } + public void testPayloadLengthPatched() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + + // Payload length is at offset 8 (4 magic + 1 version + 1 flags + 2 tablecount) + QwpBufferWriter buf = encoder.getBuffer(); + int payloadLength = Unsafe.getUnsafe().getInt(buf.getBufferPtr() + 8); + + // Payload length should be total size minus header (12 bytes) + Assert.assertEquals(size - 12, payloadLength); + } + }); } @Test - public void testReset() { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - QwpTableBuffer buffer = new QwpTableBuffer("test"); + public void testReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(1L); - buffer.nextRow(); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); - int size1 = encoder.encode(buffer, false); + int size1 = encoder.encode(buffer, false); - // Reset and encode again - buffer.reset(); - col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(2L); - buffer.nextRow(); + // Reset and encode again + buffer.reset(); + col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(2L); + buffer.nextRow(); - int size2 = encoder.encode(buffer, false); + int size2 = encoder.encode(buffer, false); - // Sizes should be similar (same schema) - Assert.assertEquals(size1, size2); - } + // Sizes should be similar (same schema) + Assert.assertEquals(size1, size2); + } + }); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index 4be668c..60a07e0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -28,7 +28,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.test.AbstractTest; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -47,7 +47,7 @@ public class QwpWebSocketSenderStateTest extends AbstractTest { @Test public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( "localhost", 0, 1, 10_000_000, 0, 1 ); @@ -89,7 +89,7 @@ public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { @Test public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( "localhost", 0, 1, 10_000_000, 0, 1 ); @@ -124,7 +124,7 @@ public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Except @Test public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { // Use high autoFlushRows to prevent auto-flush during the test QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( "localhost", 0, 10_000, 10_000_000, 0, 1 diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 6ce9885..2fccdc5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -32,6 +32,7 @@ import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.network.PlainSocketFactory; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -48,337 +49,388 @@ public class QwpWebSocketSenderTest { @Test - public void testAtAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testAtAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.at(1000L, ChronoUnit.MICROS); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testAtInstantAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testAtInstantAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.at(Instant.now()); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.at(Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testAtNowAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testAtNowAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.atNow(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testBoolColumnAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testBoolColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.boolColumn("x", true); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.boolColumn("x", true); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testBufferViewNotSupported() { - try (QwpWebSocketSender sender = createUnconnectedSender()) { - sender.bufferView(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("not supported")); - } + public void testBufferViewNotSupported() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.bufferView(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("not supported")); + } + }); } @Test - public void testCancelRowAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testCancelRowAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.cancelRow(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.cancelRow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testCancelRowDiscardsPartialRow() { - try (QwpWebSocketSender sender = createUnconnectedSender()) { - sender.table("test"); - sender.longColumn("x", 1); - sender.boolColumn("y", true); - - // Row is not yet committed (no at/atNow call), cancel it - sender.cancelRow(); - - // Buffer should have no committed rows - QwpTableBuffer buf = sender.getTableBuffer("test"); - Assert.assertEquals(0, buf.getRowCount()); - } + public void testCancelRowDiscardsPartialRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.table("test"); + sender.longColumn("x", 1); + sender.boolColumn("y", true); + + // Row is not yet committed (no at/atNow call), cancel it + sender.cancelRow(); + + // Buffer should have no committed rows + QwpTableBuffer buf = sender.getTableBuffer("test"); + Assert.assertEquals(0, buf.getRowCount()); + } + }); } @Test - public void testCancelRowNoOpWithoutTable() { - try (QwpWebSocketSender sender = createUnconnectedSender()) { - // cancelRow without table() should be a no-op (no NPE) - sender.cancelRow(); - } + public void testCancelRowNoOpWithoutTable() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + // cancelRow without table() should be a no-op (no NPE) + sender.cancelRow(); + } + }); } @Test - public void testCloseIdemponent() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); - sender.close(); // Should not throw + public void testCloseIdemponent() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + sender.close(); // Should not throw + }); } @Test - public void testConnectToClosedPort() { - try { - QwpWebSocketSender.connect("127.0.0.1", 1); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("Failed to connect")); - } + public void testConnectToClosedPort() throws Exception { + assertMemoryLeak(() -> { + try { + QwpWebSocketSender.connect("127.0.0.1", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Failed to connect")); + } + }); } @Test - public void testDoubleArrayAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testDoubleArrayAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.doubleArray("x", new double[]{1.0, 2.0}); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.doubleArray("x", new double[]{1.0, 2.0}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testDoubleColumnAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testDoubleColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.doubleColumn("x", 1.0); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.doubleColumn("x", 1.0); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testGorillaEnabledByDefault() { - try (QwpWebSocketSender sender = createUnconnectedSender()) { - Assert.assertTrue(sender.isGorillaEnabled()); - } + public void testGorillaEnabledByDefault() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + Assert.assertTrue(sender.isGorillaEnabled()); + } + }); } @Test - public void testLongArrayAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testLongArrayAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.longArray("x", new long[]{1L, 2L}); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.longArray("x", new long[]{1L, 2L}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testLongColumnAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testLongColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.longColumn("x", 1); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testNullArrayReturnsThis() { - try (QwpWebSocketSender sender = createUnconnectedSender()) { - // Null arrays should be no-ops and return sender - Assert.assertSame(sender, sender.doubleArray("x", (double[]) null)); - Assert.assertSame(sender, sender.longArray("x", (long[]) null)); - } + public void testNullArrayReturnsThis() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + // Null arrays should be no-ops and return sender + Assert.assertSame(sender, sender.doubleArray("x", (double[]) null)); + Assert.assertSame(sender, sender.longArray("x", (long[]) null)); + } + }); } @Test - public void testOperationsAfterCloseThrow() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testOperationsAfterCloseThrow() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.table("test"); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.table("test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testResetAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testResetAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.reset(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.reset(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test public void testSealAndSwapRollsBackOnEnqueueFailure() throws Exception { - try (QwpWebSocketSender sender = createUnconnectedAsyncSender(); ThrowingOnceWebSocketSendQueue queue = new ThrowingOnceWebSocketSendQueue()) { - setSendQueue(sender, queue); - - MicrobatchBuffer originalActive = getActiveBuffer(sender); - originalActive.writeByte((byte) 7); - originalActive.incrementRowCount(); - - try { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedAsyncSender(); ThrowingOnceWebSocketSendQueue queue = new ThrowingOnceWebSocketSendQueue()) { + setSendQueue(sender, queue); + + MicrobatchBuffer originalActive = getActiveBuffer(sender); + originalActive.writeByte((byte) 7); + originalActive.incrementRowCount(); + + try { + invokeSealAndSwapBuffer(sender); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Synthetic enqueue failure")); + } + + // Failed enqueue must not strand the sealed buffer. + Assert.assertSame(originalActive, getActiveBuffer(sender)); + Assert.assertTrue(originalActive.isFilling()); + Assert.assertTrue(originalActive.hasData()); + Assert.assertEquals(1, originalActive.getRowCount()); + + // Retry should be possible on the same sender instance. invokeSealAndSwapBuffer(sender); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("Synthetic enqueue failure")); + Assert.assertNotSame(originalActive, getActiveBuffer(sender)); } - - // Failed enqueue must not strand the sealed buffer. - Assert.assertSame(originalActive, getActiveBuffer(sender)); - Assert.assertTrue(originalActive.isFilling()); - Assert.assertTrue(originalActive.hasData()); - Assert.assertEquals(1, originalActive.getRowCount()); - - // Retry should be possible on the same sender instance. - invokeSealAndSwapBuffer(sender); - Assert.assertNotSame(originalActive, getActiveBuffer(sender)); - } + }); } @Test - public void testSetGorillaEnabled() { - try (QwpWebSocketSender sender = createUnconnectedSender()) { - sender.setGorillaEnabled(false); - Assert.assertFalse(sender.isGorillaEnabled()); - sender.setGorillaEnabled(true); - Assert.assertTrue(sender.isGorillaEnabled()); - } + public void testSetGorillaEnabled() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.setGorillaEnabled(false); + Assert.assertFalse(sender.isGorillaEnabled()); + sender.setGorillaEnabled(true); + Assert.assertTrue(sender.isGorillaEnabled()); + } + }); } @Test - public void testStringColumnAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testStringColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.stringColumn("x", "test"); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.stringColumn("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testSymbolAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testSymbolAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.symbol("x", "test"); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.symbol("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testTableBeforeAtNowRequired() { - try { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.atNow(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("table()")); - } + public void testTableBeforeAtNowRequired() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + }); } @Test - public void testTableBeforeAtRequired() { - try { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.at(1000L, ChronoUnit.MICROS); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("table()")); - } + public void testTableBeforeAtRequired() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + }); } @Test - public void testTableBeforeColumnsRequired() { - // Create sender without connecting (we'll catch the error earlier) - try { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.longColumn("x", 1); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("table()")); - } + public void testTableBeforeColumnsRequired() throws Exception { + assertMemoryLeak(() -> { + // Create sender without connecting (we'll catch the error earlier) + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + }); } @Test - public void testTimestampColumnAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testTimestampColumnAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.timestampColumn("x", 1000L, ChronoUnit.MICROS); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.timestampColumn("x", 1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } @Test - public void testTimestampColumnInstantAfterCloseThrows() { - QwpWebSocketSender sender = createUnconnectedSender(); - sender.close(); + public void testTimestampColumnInstantAfterCloseThrows() throws Exception { + assertMemoryLeak(() -> { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); - try { - sender.timestampColumn("x", Instant.now()); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("closed")); - } + try { + sender.timestampColumn("x", Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + }); } private static MicrobatchBuffer getActiveBuffer(QwpWebSocketSender sender) throws Exception { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java index 4629272..022bfde 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java @@ -34,6 +34,7 @@ import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import java.util.concurrent.CountDownLatch; @@ -46,211 +47,223 @@ public class WebSocketSendQueueTest { @Test - public void testEnqueueTimeoutWhenPendingSlotOccupied() { - InFlightWindow window = new InFlightWindow(1, 1_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - MicrobatchBuffer batch0 = sealedBuffer((byte) 1); - MicrobatchBuffer batch1 = sealedBuffer((byte) 2); - WebSocketSendQueue queue = null; - - try { - // Keep window full so I/O thread cannot drain pending slot. - window.addInFlight(0); - queue = new WebSocketSendQueue(client, window, 100, 500); - queue.enqueue(batch0); + public void testEnqueueTimeoutWhenPendingSlotOccupied() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; try { - queue.enqueue(batch1); - fail("Expected enqueue timeout"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Enqueue timeout")); + // Keep window full so I/O thread cannot drain pending slot. + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 100, 500); + queue.enqueue(batch0); + + try { + queue.enqueue(batch1); + fail("Expected enqueue timeout"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Enqueue timeout")); + } + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); } - } finally { - window.acknowledgeUpTo(Long.MAX_VALUE); - closeQuietly(queue); - batch0.close(); - batch1.close(); - client.close(); - } + }); } @Test public void testEnqueueWaitsUntilSlotAvailable() throws Exception { - InFlightWindow window = new InFlightWindow(1, 1_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - MicrobatchBuffer batch0 = sealedBuffer((byte) 1); - MicrobatchBuffer batch1 = sealedBuffer((byte) 2); - WebSocketSendQueue queue = null; - - try { - window.addInFlight(0); - queue = new WebSocketSendQueue(client, window, 2_000, 500); - final WebSocketSendQueue finalQueue = queue; - queue.enqueue(batch0); - - CountDownLatch started = new CountDownLatch(1); - CountDownLatch finished = new CountDownLatch(1); - AtomicReference errorRef = new AtomicReference<>(); - - Thread t = new Thread(() -> { - started.countDown(); - try { - finalQueue.enqueue(batch1); - } catch (Throwable t1) { - errorRef.set(t1); - } finally { - finished.countDown(); - } - }); - t.start(); - - assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); - assertEquals("Second enqueue should still be waiting", 1, finished.getCount()); + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; - // Free space so I/O thread can poll pending slot. - window.acknowledgeUpTo(0); - - assertTrue("Second enqueue should complete", finished.await(2, TimeUnit.SECONDS)); - assertNull(errorRef.get()); - } finally { - window.acknowledgeUpTo(Long.MAX_VALUE); - closeQuietly(queue); - batch0.close(); - batch1.close(); - client.close(); - } + try { + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 2_000, 500); + final WebSocketSendQueue finalQueue = queue; + queue.enqueue(batch0); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + AtomicReference errorRef = new AtomicReference<>(); + + Thread t = new Thread(() -> { + started.countDown(); + try { + finalQueue.enqueue(batch1); + } catch (Throwable t1) { + errorRef.set(t1); + } finally { + finished.countDown(); + } + }); + t.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertEquals("Second enqueue should still be waiting", 1, finished.getCount()); + + // Free space so I/O thread can poll pending slot. + window.acknowledgeUpTo(0); + + assertTrue("Second enqueue should complete", finished.await(2, TimeUnit.SECONDS)); + assertNull(errorRef.get()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + }); } @Test public void testFlushFailsOnInvalidAckPayload() throws Exception { - InFlightWindow window = new InFlightWindow(8, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - WebSocketSendQueue queue = null; - CountDownLatch payloadDelivered = new CountDownLatch(1); - AtomicBoolean fired = new AtomicBoolean(false); - - try { - window.addInFlight(0); - client.setTryReceiveBehavior(handler -> { - if (fired.compareAndSet(false, true)) { - emitBinary(handler, new byte[]{1, 2, 3}); - payloadDelivered.countDown(); - return true; - } - return false; - }); - - queue = new WebSocketSendQueue(client, window, 1_000, 500); - assertTrue("Expected invalid payload callback", payloadDelivered.await(2, TimeUnit.SECONDS)); + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch payloadDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); try { - queue.flush(); - fail("Expected flush failure on invalid payload"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Invalid ACK response payload")); + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + emitBinary(handler, new byte[]{1, 2, 3}); + payloadDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected invalid payload callback", payloadDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure on invalid payload"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Invalid ACK response payload")); + } + } finally { + closeQuietly(queue); + client.close(); } - } finally { - closeQuietly(queue); - client.close(); - } + }); } @Test public void testFlushFailsOnReceiveIoError() throws Exception { - InFlightWindow window = new InFlightWindow(8, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - WebSocketSendQueue queue = null; - CountDownLatch receiveAttempted = new CountDownLatch(1); - - try { - window.addInFlight(0); - client.setTryReceiveBehavior(handler -> { - receiveAttempted.countDown(); - throw new RuntimeException("recv-fail"); - }); - - queue = new WebSocketSendQueue(client, window, 1_000, 500); - assertTrue("Expected receive attempt", receiveAttempted.await(2, TimeUnit.SECONDS)); - long deadline = System.currentTimeMillis() + 2_000; - while (queue.getLastError() == null && System.currentTimeMillis() < deadline) { - Thread.sleep(5); - } - assertNotNull("Expected queue error after receive failure", queue.getLastError()); + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch receiveAttempted = new CountDownLatch(1); try { - queue.flush(); - fail("Expected flush failure after receive error"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Error receiving response")); + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + receiveAttempted.countDown(); + throw new RuntimeException("recv-fail"); + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected receive attempt", receiveAttempted.await(2, TimeUnit.SECONDS)); + long deadline = System.currentTimeMillis() + 2_000; + while (queue.getLastError() == null && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + assertNotNull("Expected queue error after receive failure", queue.getLastError()); + + try { + queue.flush(); + fail("Expected flush failure after receive error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Error receiving response")); + } + } finally { + closeQuietly(queue); + client.close(); } - } finally { - closeQuietly(queue); - client.close(); - } + }); } @Test - public void testFlushFailsOnSendIoError() { - FakeWebSocketClient client = new FakeWebSocketClient(); - MicrobatchBuffer batch = sealedBuffer((byte) 42); - WebSocketSendQueue queue = null; - - try { - client.setSendBehavior((dataPtr, length) -> { - throw new RuntimeException("send-fail"); - }); - queue = new WebSocketSendQueue(client, null, 1_000, 500); - queue.enqueue(batch); + public void testFlushFailsOnSendIoError() throws Exception { + assertMemoryLeak(() -> { + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch = sealedBuffer((byte) 42); + WebSocketSendQueue queue = null; try { - queue.flush(); - fail("Expected flush failure after send error"); - } catch (LineSenderException e) { - assertTrue( - e.getMessage().contains("Error sending batch") - || e.getMessage().contains("Error in send queue I/O thread") - ); + client.setSendBehavior((dataPtr, length) -> { + throw new RuntimeException("send-fail"); + }); + queue = new WebSocketSendQueue(client, null, 1_000, 500); + queue.enqueue(batch); + + try { + queue.flush(); + fail("Expected flush failure after send error"); + } catch (LineSenderException e) { + assertTrue( + e.getMessage().contains("Error sending batch") + || e.getMessage().contains("Error in send queue I/O thread") + ); + } + } finally { + closeQuietly(queue); + batch.close(); + client.close(); } - } finally { - closeQuietly(queue); - batch.close(); - client.close(); - } + }); } @Test public void testFlushFailsWhenServerClosesConnection() throws Exception { - InFlightWindow window = new InFlightWindow(8, 5_000); - FakeWebSocketClient client = new FakeWebSocketClient(); - WebSocketSendQueue queue = null; - CountDownLatch closeDelivered = new CountDownLatch(1); - AtomicBoolean fired = new AtomicBoolean(false); - - try { - window.addInFlight(0); - client.setTryReceiveBehavior(handler -> { - if (fired.compareAndSet(false, true)) { - handler.onClose(1006, "boom"); - closeDelivered.countDown(); - return true; - } - return false; - }); - - queue = new WebSocketSendQueue(client, window, 1_000, 500); - assertTrue("Expected close callback", closeDelivered.await(2, TimeUnit.SECONDS)); + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch closeDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); try { - queue.flush(); - fail("Expected flush failure after close"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("closed")); + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + handler.onClose(1006, "boom"); + closeDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected close callback", closeDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure after close"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("closed")); + } + } finally { + closeQuietly(queue); + client.close(); } - } finally { - closeQuietly(queue); - client.close(); - } + }); } private static void closeQuietly(WebSocketSendQueue queue) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java index d7755f4..8726bcf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -35,322 +36,370 @@ public class OffHeapAppendMemoryTest { @Test - public void testCloseFreesMemory() { - long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); - long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertTrue(during > before); - - mem.close(); - long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertEquals(before, after); + public void testCloseFreesMemory() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); + long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertTrue(during > before); + + mem.close(); + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); } @Test - public void testDoubleCloseIsSafe() { - OffHeapAppendMemory mem = new OffHeapAppendMemory(); - mem.putInt(42); - mem.close(); - mem.close(); // should not throw + public void testDoubleCloseIsSafe() throws Exception { + assertMemoryLeak(() -> { + OffHeapAppendMemory mem = new OffHeapAppendMemory(); + mem.putInt(42); + mem.close(); + mem.close(); // should not throw + }); } @Test - public void testGrowth() { - long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { - // Write more data than initial capacity to force growth - for (int i = 0; i < 100; i++) { - mem.putLong(i); - } - - assertEquals(800, mem.getAppendOffset()); - for (int i = 0; i < 100; i++) { - assertEquals(i, Unsafe.getUnsafe().getLong(mem.addressOf((long) i * 8))); + public void testGrowth() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write more data than initial capacity to force growth + for (int i = 0; i < 100; i++) { + mem.putLong(i); + } + + assertEquals(800, mem.getAppendOffset()); + for (int i = 0; i < 100; i++) { + assertEquals(i, Unsafe.getUnsafe().getLong(mem.addressOf((long) i * 8))); + } } - } - long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertEquals(before, after); + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); } @Test - public void testJumpTo() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putLong(100); - mem.putLong(200); - mem.putLong(300); - assertEquals(24, mem.getAppendOffset()); - - // Jump back to offset 8 (after first long) - mem.jumpTo(8); - assertEquals(8, mem.getAppendOffset()); - - // Write new value at offset 8 - mem.putLong(999); - assertEquals(16, mem.getAppendOffset()); - assertEquals(100, Unsafe.getUnsafe().getLong(mem.addressOf(0))); - assertEquals(999, Unsafe.getUnsafe().getLong(mem.addressOf(8))); - } + public void testJumpTo() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(100); + mem.putLong(200); + mem.putLong(300); + assertEquals(24, mem.getAppendOffset()); + + // Jump back to offset 8 (after first long) + mem.jumpTo(8); + assertEquals(8, mem.getAppendOffset()); + + // Write new value at offset 8 + mem.putLong(999); + assertEquals(16, mem.getAppendOffset()); + assertEquals(100, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(999, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + }); } @Test - public void testLargeGrowth() { - long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { - // Write 10000 doubles to stress growth - for (int i = 0; i < 10_000; i++) { - mem.putDouble(i * 1.1); + public void testLargeGrowth() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write 10000 doubles to stress growth + for (int i = 0; i < 10_000; i++) { + mem.putDouble(i * 1.1); + } + assertEquals(80_000, mem.getAppendOffset()); + + // Verify first and last values + assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); } - assertEquals(80_000, mem.getAppendOffset()); - - // Verify first and last values - assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); - assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); - } - long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertEquals(before, after); + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); } @Test - public void testMixedTypes() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putByte((byte) 1); - mem.putShort((short) 2); - mem.putInt(3); - mem.putLong(4L); - mem.putFloat(5.0f); - mem.putDouble(6.0); - - long addr = mem.pageAddress(); - assertEquals(1, Unsafe.getUnsafe().getByte(addr)); - assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); - assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); - assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); - assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); - assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); - assertEquals(27, mem.getAppendOffset()); - } + public void testMixedTypes() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 1); + mem.putShort((short) 2); + mem.putInt(3); + mem.putLong(4L); + mem.putFloat(5.0f); + mem.putDouble(6.0); + + long addr = mem.pageAddress(); + assertEquals(1, Unsafe.getUnsafe().getByte(addr)); + assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); + assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); + assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); + assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); + assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); + assertEquals(27, mem.getAppendOffset()); + } + }); } @Test - public void testPageAddress() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - assertTrue(mem.pageAddress() != 0); - assertEquals(mem.pageAddress(), mem.addressOf(0)); - mem.putLong(42); - assertEquals(mem.pageAddress() + 8, mem.addressOf(8)); - } + public void testPageAddress() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + assertTrue(mem.pageAddress() != 0); + assertEquals(mem.pageAddress(), mem.addressOf(0)); + mem.putLong(42); + assertEquals(mem.pageAddress() + 8, mem.addressOf(8)); + } + }); } @Test - public void testPutAndReadByte() { - long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putByte((byte) 42); - mem.putByte((byte) -1); - mem.putByte((byte) 0); - - assertEquals(3, mem.getAppendOffset()); - assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - } - long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertEquals(before, after); + public void testPutAndReadByte() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 42); + mem.putByte((byte) -1); + mem.putByte((byte) 0); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); } @Test - public void testPutAndReadDouble() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putDouble(2.718281828); - mem.putDouble(Double.NaN); - - assertEquals(16, mem.getAppendOffset()); - assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); - assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); - } + public void testPutAndReadDouble() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putDouble(2.718281828); + mem.putDouble(Double.NaN); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); + } + }); } @Test - public void testPutAndReadFloat() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putFloat(3.14f); - mem.putFloat(Float.NaN); - - assertEquals(8, mem.getAppendOffset()); - assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); - assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); - } + public void testPutAndReadFloat() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putFloat(3.14f); + mem.putFloat(Float.NaN); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); + assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); + } + }); } @Test - public void testPutAndReadInt() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(100_000); - mem.putInt(Integer.MIN_VALUE); - - assertEquals(8, mem.getAppendOffset()); - assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); - } + public void testPutAndReadInt() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(100_000); + mem.putInt(Integer.MIN_VALUE); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); + } + }); } @Test - public void testPutAndReadLong() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putLong(1_000_000_000_000L); - mem.putLong(Long.MIN_VALUE); - - assertEquals(16, mem.getAppendOffset()); - assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); - assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); - } + public void testPutAndReadLong() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(1_000_000_000_000L); + mem.putLong(Long.MIN_VALUE); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + }); } @Test - public void testPutAndReadShort() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putShort((short) 12_345); - mem.putShort(Short.MIN_VALUE); - mem.putShort(Short.MAX_VALUE); - - assertEquals(6, mem.getAppendOffset()); - assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); - assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); - assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); - } + public void testPutAndReadShort() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putShort((short) 12_345); + mem.putShort(Short.MIN_VALUE); + mem.putShort(Short.MAX_VALUE); + + assertEquals(6, mem.getAppendOffset()); + assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); + assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); + assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); + } + }); } @Test - public void testPutBoolean() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putBoolean(true); - mem.putBoolean(false); - mem.putBoolean(true); - - assertEquals(3, mem.getAppendOffset()); - assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - } + public void testPutBoolean() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putBoolean(true); + mem.putBoolean(false); + mem.putBoolean(true); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + }); } @Test - public void testPutUtf8Ascii() { - long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putUtf8("hello"); - assertEquals(5, mem.getAppendOffset()); - assertEquals('h', Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals('e', Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(2))); - assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(3))); - assertEquals('o', Unsafe.getUnsafe().getByte(mem.addressOf(4))); - } - long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertEquals(before, after); + public void testPutUtf8Ascii() throws Exception { + assertMemoryLeak(() -> { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putUtf8("hello"); + assertEquals(5, mem.getAppendOffset()); + assertEquals('h', Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals('e', Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(2))); + assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(3))); + assertEquals('o', Unsafe.getUnsafe().getByte(mem.addressOf(4))); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + }); } @Test - public void testPutUtf8Empty() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putUtf8(""); - assertEquals(0, mem.getAppendOffset()); - } + public void testPutUtf8Empty() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putUtf8(""); + assertEquals(0, mem.getAppendOffset()); + } + }); } @Test - public void testPutUtf8Mixed() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes - mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00"); - assertEquals(10, mem.getAppendOffset()); - } + public void testPutUtf8Mixed() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes + mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00"); + assertEquals(10, mem.getAppendOffset()); + } + }); } @Test - public void testPutUtf8MultiByte() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - // 2-byte: U+00E9 (e-acute) = C3 A9 - mem.putUtf8("\u00E9"); - assertEquals(2, mem.getAppendOffset()); - assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals((byte) 0xA9, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - } + public void testPutUtf8MultiByte() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // 2-byte: U+00E9 (e-acute) = C3 A9 + mem.putUtf8("\u00E9"); + assertEquals(2, mem.getAppendOffset()); + assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 0xA9, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + } + }); } @Test - public void testPutUtf8Null() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putUtf8(null); - assertEquals(0, mem.getAppendOffset()); - } + public void testPutUtf8Null() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putUtf8(null); + assertEquals(0, mem.getAppendOffset()); + } + }); } @Test - public void testPutUtf8InvalidSurrogatePair() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - // High surrogate \uD800 followed by non-low-surrogate 'X'. - // Should produce '?' for the lone high surrogate, then 'X'. - mem.putUtf8("\uD800X"); - assertEquals(2, mem.getAppendOffset()); - assertEquals((byte) '?', Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(mem.addressOf(1))); - } + public void testPutUtf8InvalidSurrogatePair() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + mem.putUtf8("\uD800X"); + assertEquals(2, mem.getAppendOffset()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(mem.addressOf(1))); + } + }); } @Test - public void testPutUtf8SurrogatePairs() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - // U+1F600 (grinning face) = F0 9F 98 80 - mem.putUtf8("\uD83D\uDE00"); - assertEquals(4, mem.getAppendOffset()); - assertEquals((byte) 0xF0, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals((byte) 0x9F, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals((byte) 0x98, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(mem.addressOf(3))); - } + public void testPutUtf8SurrogatePairs() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // U+1F600 (grinning face) = F0 9F 98 80 + mem.putUtf8("\uD83D\uDE00"); + assertEquals(4, mem.getAppendOffset()); + assertEquals((byte) 0xF0, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 0x9F, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals((byte) 0x98, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(mem.addressOf(3))); + } + }); } @Test - public void testPutUtf8ThreeByte() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - // 3-byte: U+4E16 (CJK character) = E4 B8 96 - mem.putUtf8("\u4E16"); - assertEquals(3, mem.getAppendOffset()); - assertEquals((byte) 0xE4, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals((byte) 0xB8, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals((byte) 0x96, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - } + public void testPutUtf8ThreeByte() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // 3-byte: U+4E16 (CJK character) = E4 B8 96 + mem.putUtf8("\u4E16"); + assertEquals(3, mem.getAppendOffset()); + assertEquals((byte) 0xE4, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 0xB8, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals((byte) 0x96, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + }); } @Test - public void testSkip() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(1); - mem.skip(8); - mem.putInt(2); - - assertEquals(16, mem.getAppendOffset()); - assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); - } + public void testSkip() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.skip(8); + mem.putInt(2); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + } + }); } @Test - public void testTruncate() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(1); - mem.putInt(2); - mem.putInt(3); - assertEquals(12, mem.getAppendOffset()); - - mem.truncate(); - assertEquals(0, mem.getAppendOffset()); - - // Can write again after truncate - mem.putInt(42); - assertEquals(4, mem.getAppendOffset()); - assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - } + public void testTruncate() throws Exception { + assertMemoryLeak(() -> { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.putInt(2); + mem.putInt(3); + assertEquals(12, mem.getAppendOffset()); + + mem.truncate(); + assertEquals(0, mem.getAppendOffset()); + + // Can write again after truncate + mem.putInt(42); + assertEquals(4, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + } + }); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java index 0b516e0..5ec8c60 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.*; @@ -36,154 +37,170 @@ public class QwpBitWriterTest { @Test - public void testFlushThrowsOnOverflow() { - long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 1); - // Write 8 bits to fill the single byte - writer.writeBits(0xFF, 8); - // Write a few more bits that sit in the bit buffer - writer.writeBits(0x3, 4); - // Flush should throw because there's no room for the partial byte + public void testFlushThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); try { - writer.flush(); - fail("expected LineSenderException on buffer overflow during flush"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 1); + // Write 8 bits to fill the single byte + writer.writeBits(0xFF, 8); + // Write a few more bits that sit in the bit buffer + writer.writeBits(0x3, 4); + // Flush should throw because there's no room for the partial byte + try { + writer.flush(); + fail("expected LineSenderException on buffer overflow during flush"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() { - // Source: 1 timestamp (8 bytes), dest: only 4 bytes - long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); - long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); - try { - Unsafe.getUnsafe().putLong(src, 1_000_000L); - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() throws Exception { + assertMemoryLeak(() -> { + // Source: 1 timestamp (8 bytes), dest: only 4 bytes + long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); try { - encoder.encodeTimestamps(dst, 4, src, 1); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); + Unsafe.getUnsafe().putLong(src, 1_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + try { + encoder.encodeTimestamps(dst, 4, src, 1); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() { - // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second) - long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS); - try { - Unsafe.getUnsafe().putLong(src, 1_000_000L); - Unsafe.getUnsafe().putLong(src + 8, 2_000_000L); - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() throws Exception { + assertMemoryLeak(() -> { + // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second) + long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS); try { - encoder.encodeTimestamps(dst, 12, src, 2); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); + Unsafe.getUnsafe().putLong(src, 1_000_000L); + Unsafe.getUnsafe().putLong(src + 8, 2_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + try { + encoder.encodeTimestamps(dst, 12, src, 2); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testWriteBitsThrowsOnOverflow() { - long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 4); - // Fill the buffer (32 bits = 4 bytes) - writer.writeBits(0xFFFF_FFFFL, 32); - // Next write should throw — buffer is full + public void testWriteBitsThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); try { - writer.writeBits(1, 8); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 4); + // Fill the buffer (32 bits = 4 bytes) + writer.writeBits(0xFFFF_FFFFL, 32); + // Next write should throw — buffer is full + try { + writer.writeBits(1, 8); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testWriteBitsWithinCapacitySucceeds() { - long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 8); - writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64); - writer.flush(); - assertEquals(8, writer.getPosition() - ptr); - assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr)); - } finally { - Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testWriteBitsWithinCapacitySucceeds() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); + try { + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 8); + writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64); + writer.flush(); + assertEquals(8, writer.getPosition() - ptr); + assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr)); + } finally { + Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testWriteByteThrowsOnOverflow() { - long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 1); - writer.writeByte(0x42); + public void testWriteByteThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); try { - writer.writeByte(0x43); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 1); + writer.writeByte(0x42); + try { + writer.writeByte(0x43); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testWriteIntThrowsOnOverflow() { - long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 4); - writer.writeInt(42); + public void testWriteIntThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); try { - writer.writeInt(99); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 4); + writer.writeInt(42); + try { + writer.writeInt(99); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testWriteLongThrowsOnOverflow() { - long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 8); - writer.writeLong(42L); + public void testWriteLongThrowsOnOverflow() throws Exception { + assertMemoryLeak(() -> { + long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); try { - writer.writeLong(99L); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 8); + writer.writeLong(42L); + try { + writer.writeLong(99L); + fail("expected LineSenderException on buffer overflow"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); - } + }); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java index 0e2e553..09a46dd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java @@ -28,589 +28,650 @@ import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; public class QwpGorillaEncoderTest { @Test - public void testCalculateEncodedSizeConstantDelta() { - long[] timestamps = new long[100]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = 1_000_000_000L + i * 1000L; - } - long src = putTimestamps(timestamps); - try { - int size = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); - // 8 (first) + 8 (second) + ceil(98 bits / 8) = 29 - int expectedBits = timestamps.length - 2; - int expectedSize = 8 + 8 + (expectedBits + 7) / 8; - Assert.assertEquals(expectedSize, size); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCalculateEncodedSizeConstantDelta() throws Exception { + assertMemoryLeak(() -> { + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + long src = putTimestamps(timestamps); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + // 8 (first) + 8 (second) + ceil(98 bits / 8) = 29 + int expectedBits = timestamps.length - 2; + int expectedSize = 8 + 8 + (expectedBits + 7) / 8; + Assert.assertEquals(expectedSize, size); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCalculateEncodedSizeEmpty() { - Assert.assertEquals(0, QwpGorillaEncoder.calculateEncodedSize(0, 0)); + public void testCalculateEncodedSizeEmpty() throws Exception { + assertMemoryLeak(() -> { + Assert.assertEquals(0, QwpGorillaEncoder.calculateEncodedSize(0, 0)); + }); } @Test - public void testCalculateEncodedSizeIdenticalDeltas() { - long[] ts = {100L, 200L, 300L}; // delta=100, DoD=0 - long src = putTimestamps(ts); - try { - int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); - // 8 + 8 + 1 byte (1 bit padded to byte) = 17 - Assert.assertEquals(17, size); - } finally { - Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCalculateEncodedSizeIdenticalDeltas() throws Exception { + assertMemoryLeak(() -> { + long[] ts = {100L, 200L, 300L}; // delta=100, DoD=0 + long src = putTimestamps(ts); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); + // 8 + 8 + 1 byte (1 bit padded to byte) = 17 + Assert.assertEquals(17, size); + } finally { + Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCalculateEncodedSizeOneTimestamp() { - long[] ts = {1000L}; - long src = putTimestamps(ts); - try { - Assert.assertEquals(8, QwpGorillaEncoder.calculateEncodedSize(src, 1)); - } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCalculateEncodedSizeOneTimestamp() throws Exception { + assertMemoryLeak(() -> { + long[] ts = {1000L}; + long src = putTimestamps(ts); + try { + Assert.assertEquals(8, QwpGorillaEncoder.calculateEncodedSize(src, 1)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCalculateEncodedSizeSmallDoD() { - long[] ts = {100L, 200L, 350L}; // delta0=100, delta1=150, DoD=50 - long src = putTimestamps(ts); - try { - int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); - // 8 + 8 + 2 bytes (9 bits padded to bytes) = 18 - Assert.assertEquals(18, size); - } finally { - Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCalculateEncodedSizeSmallDoD() throws Exception { + assertMemoryLeak(() -> { + long[] ts = {100L, 200L, 350L}; // delta0=100, delta1=150, DoD=50 + long src = putTimestamps(ts); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); + // 8 + 8 + 2 bytes (9 bits padded to bytes) = 18 + Assert.assertEquals(18, size); + } finally { + Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCalculateEncodedSizeTwoTimestamps() { - long[] ts = {1000L, 2000L}; - long src = putTimestamps(ts); - try { - Assert.assertEquals(16, QwpGorillaEncoder.calculateEncodedSize(src, 2)); - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - } + public void testCalculateEncodedSizeTwoTimestamps() throws Exception { + assertMemoryLeak(() -> { + long[] ts = {1000L, 2000L}; + long src = putTimestamps(ts); + try { + Assert.assertEquals(16, QwpGorillaEncoder.calculateEncodedSize(src, 2)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCanUseGorillaConstantDelta() { - long[] timestamps = new long[100]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = 1_000_000_000L + i * 1000L; - } - long src = putTimestamps(timestamps); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCanUseGorillaConstantDelta() throws Exception { + assertMemoryLeak(() -> { + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + long src = putTimestamps(timestamps); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCanUseGorillaEmpty() { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(0, 0)); + public void testCanUseGorillaEmpty() throws Exception { + assertMemoryLeak(() -> { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(0, 0)); + }); } @Test - public void testCanUseGorillaLargeDoDOutOfRange() { - // DoD = 3_000_000_000 exceeds Integer.MAX_VALUE - long[] timestamps = { - 0L, - 1_000_000_000L, // delta=1_000_000_000 - 5_000_000_000L, // delta=4_000_000_000, DoD=3_000_000_000 - }; - long src = putTimestamps(timestamps); - try { - Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCanUseGorillaLargeDoDOutOfRange() throws Exception { + assertMemoryLeak(() -> { + // DoD = 3_000_000_000 exceeds Integer.MAX_VALUE + long[] timestamps = { + 0L, + 1_000_000_000L, // delta=1_000_000_000 + 5_000_000_000L, // delta=4_000_000_000, DoD=3_000_000_000 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCanUseGorillaNegativeLargeDoDOutOfRange() { - // DoD = -4_000_000_000 is less than Integer.MIN_VALUE - long[] timestamps = { - 10_000_000_000L, - 9_000_000_000L, // delta=-1_000_000_000 - 4_000_000_000L, // delta=-5_000_000_000, DoD=-4_000_000_000 - }; - long src = putTimestamps(timestamps); - try { - Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCanUseGorillaNegativeLargeDoDOutOfRange() throws Exception { + assertMemoryLeak(() -> { + // DoD = -4_000_000_000 is less than Integer.MIN_VALUE + long[] timestamps = { + 10_000_000_000L, + 9_000_000_000L, // delta=-1_000_000_000 + 4_000_000_000L, // delta=-5_000_000_000, DoD=-4_000_000_000 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCanUseGorillaOneTimestamp() { - long[] ts = {1000L}; - long src = putTimestamps(ts); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 1)); - } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCanUseGorillaOneTimestamp() throws Exception { + assertMemoryLeak(() -> { + long[] ts = {1000L}; + long src = putTimestamps(ts); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 1)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCanUseGorillaTwoTimestamps() { - long[] ts = {1000L, 2000L}; - long src = putTimestamps(ts); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 2)); - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - } + public void testCanUseGorillaTwoTimestamps() throws Exception { + assertMemoryLeak(() -> { + long[] ts = {1000L, 2000L}; + long src = putTimestamps(ts); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 2)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCanUseGorillaVaryingDelta() { - long[] timestamps = { - 1_000_000_000L, - 1_000_001_000L, // delta=1000 - 1_000_002_100L, // DoD=100 - 1_000_003_500L, // DoD=300 - }; - long src = putTimestamps(timestamps); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCanUseGorillaVaryingDelta() throws Exception { + assertMemoryLeak(() -> { + long[] timestamps = { + 1_000_000_000L, + 1_000_001_000L, // delta=1000 + 1_000_002_100L, // DoD=100 + 1_000_003_500L, // DoD=300 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCompressionRatioConstantInterval() { - long[] timestamps = new long[1000]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = i * 1000L; - } - long src = putTimestamps(timestamps); - try { - int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); - int uncompressedSize = timestamps.length * 8; - double ratio = (double) gorillaSize / uncompressedSize; - Assert.assertTrue("Compression ratio should be < 0.1 for constant interval", ratio < 0.1); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCompressionRatioConstantInterval() throws Exception { + assertMemoryLeak(() -> { + long[] timestamps = new long[1000]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = i * 1000L; + } + long src = putTimestamps(timestamps); + try { + int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + int uncompressedSize = timestamps.length * 8; + double ratio = (double) gorillaSize / uncompressedSize; + Assert.assertTrue("Compression ratio should be < 0.1 for constant interval", ratio < 0.1); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testCompressionRatioRandomData() { - long[] timestamps = new long[100]; - timestamps[0] = 1_000_000_000L; - timestamps[1] = 1_000_001_000L; - java.util.Random random = new java.util.Random(42); - for (int i = 2; i < timestamps.length; i++) { - timestamps[i] = timestamps[i - 1] + 1000 + random.nextInt(10_000) - 5000; - } - long src = putTimestamps(timestamps); - try { - int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); - Assert.assertTrue("Size should be positive", gorillaSize > 0); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } + public void testCompressionRatioRandomData() throws Exception { + assertMemoryLeak(() -> { + long[] timestamps = new long[100]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; + java.util.Random random = new java.util.Random(42); + for (int i = 2; i < timestamps.length; i++) { + timestamps[i] = timestamps[i - 1] + 1000 + random.nextInt(10_000) - 5000; + } + long src = putTimestamps(timestamps); + try { + int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + Assert.assertTrue("Size should be positive", gorillaSize > 0); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + }); + } + + @Test + public void testEncodeDecodeBucketBoundaries() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + // t0=0, t1=10_000, delta0=10_000 + // For DoD=X: delta1 = 10_000+X, so t2 = 20_000+X + long[][] bucketTests = { + {0L, 10_000L, 20_000L}, // DoD = 0 (bucket 0) + {0L, 10_000L, 20_063L}, // DoD = 63 (bucket 1 max) + {0L, 10_000L, 19_936L}, // DoD = -64 (bucket 1 min) + {0L, 10_000L, 20_064L}, // DoD = 64 (bucket 2 start) + {0L, 10_000L, 19_935L}, // DoD = -65 (bucket 2 start) + {0L, 10_000L, 20_255L}, // DoD = 255 (bucket 2 max) + {0L, 10_000L, 19_744L}, // DoD = -256 (bucket 2 min) + {0L, 10_000L, 20_256L}, // DoD = 256 (bucket 3 start) + {0L, 10_000L, 19_743L}, // DoD = -257 (bucket 3 start) + {0L, 10_000L, 22_047L}, // DoD = 2047 (bucket 3 max) + {0L, 10_000L, 17_952L}, // DoD = -2048 (bucket 3 min) + {0L, 10_000L, 22_048L}, // DoD = 2048 (bucket 4 start) + {0L, 10_000L, 17_951L}, // DoD = -2049 (bucket 4 start) + {0L, 10_000L, 110_000L}, // DoD = 100_000 (bucket 4, large) + {0L, 10_000L, -80_000L}, // DoD = -100_000 (bucket 4, large) + }; + + for (long[] tc : bucketTests) { + long src = putTimestamps(tc); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, tc.length); + Assert.assertTrue("Failed to encode: " + java.util.Arrays.toString(tc), bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(tc[0], first); + Assert.assertEquals(tc[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long dod = decodeDoD(reader); + long delta = (second - first) + dod; + long decoded = second + delta; + Assert.assertEquals("Failed for: " + java.util.Arrays.toString(tc), tc[2], decoded); + } finally { + Unsafe.free(src, (long) tc.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + }); } @Test - public void testEncodeDecodeBucketBoundaries() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - // t0=0, t1=10_000, delta0=10_000 - // For DoD=X: delta1 = 10_000+X, so t2 = 20_000+X - long[][] bucketTests = { - {0L, 10_000L, 20_000L}, // DoD = 0 (bucket 0) - {0L, 10_000L, 20_063L}, // DoD = 63 (bucket 1 max) - {0L, 10_000L, 19_936L}, // DoD = -64 (bucket 1 min) - {0L, 10_000L, 20_064L}, // DoD = 64 (bucket 2 start) - {0L, 10_000L, 19_935L}, // DoD = -65 (bucket 2 start) - {0L, 10_000L, 20_255L}, // DoD = 255 (bucket 2 max) - {0L, 10_000L, 19_744L}, // DoD = -256 (bucket 2 min) - {0L, 10_000L, 20_256L}, // DoD = 256 (bucket 3 start) - {0L, 10_000L, 19_743L}, // DoD = -257 (bucket 3 start) - {0L, 10_000L, 22_047L}, // DoD = 2047 (bucket 3 max) - {0L, 10_000L, 17_952L}, // DoD = -2048 (bucket 3 min) - {0L, 10_000L, 22_048L}, // DoD = 2048 (bucket 4 start) - {0L, 10_000L, 17_951L}, // DoD = -2049 (bucket 4 start) - {0L, 10_000L, 110_000L}, // DoD = 100_000 (bucket 4, large) - {0L, 10_000L, -80_000L}, // DoD = -100_000 (bucket 4, large) - }; - - for (long[] tc : bucketTests) { - long src = putTimestamps(tc); + public void testEncodeTimestampsEmpty() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, src, tc.length); - Assert.assertTrue("Failed to encode: " + java.util.Arrays.toString(tc), bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(tc[0], first); - Assert.assertEquals(tc[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long dod = decodeDoD(reader); - long delta = (second - first) + dod; - long decoded = second + delta; - Assert.assertEquals("Failed for: " + java.util.Arrays.toString(tc), tc[2], decoded); + int bytesWritten = encoder.encodeTimestamps(dst, 64, 0, 0); + Assert.assertEquals(0, bytesWritten); } finally { - Unsafe.free(src, (long) tc.length * 8, MemoryTag.NATIVE_ILP_RSS); Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); } - } + }); } @Test - public void testEncodeTimestampsEmpty() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, 0, 0); - Assert.assertEquals(0, bytesWritten); - } finally { - Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); - } + public void testEncodeTimestampsOneTimestamp() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long[] timestamps = {1_234_567_890L}; + long src = putTimestamps(timestamps); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 1); + Assert.assertEquals(8, bytesWritten); + Assert.assertEquals(1_234_567_890L, Unsafe.getUnsafe().getLong(dst)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testEncodeTimestampsOneTimestamp() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - long[] timestamps = {1_234_567_890L}; - long src = putTimestamps(timestamps); - long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 1); - Assert.assertEquals(8, bytesWritten); - Assert.assertEquals(1_234_567_890L, Unsafe.getUnsafe().getLong(dst)); - } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); - } - } + public void testEncodeTimestampsRoundTripAllBuckets() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = new long[10]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; // delta = 1000 + timestamps[2] = 1_000_002_000L; // DoD=0 (bucket 0) + timestamps[3] = 1_000_003_050L; // DoD=50 (bucket 1) + timestamps[4] = 1_000_003_987L; // DoD=-113 (bucket 2) + timestamps[5] = 1_000_004_687L; // DoD=-237 (bucket 2) + timestamps[6] = 1_000_006_387L; // DoD=1000 (bucket 3) + timestamps[7] = 1_000_020_087L; // DoD=12000 (bucket 4) + timestamps[8] = 1_000_033_787L; // DoD=0 (bucket 0) + timestamps[9] = 1_000_047_487L; // DoD=0 (bucket 0) + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); - @Test - public void testEncodeTimestampsRoundTripAllBuckets() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - long[] timestamps = new long[10]; - timestamps[0] = 1_000_000_000L; - timestamps[1] = 1_000_001_000L; // delta = 1000 - timestamps[2] = 1_000_002_000L; // DoD=0 (bucket 0) - timestamps[3] = 1_000_003_050L; // DoD=50 (bucket 1) - timestamps[4] = 1_000_003_987L; // DoD=-113 (bucket 2) - timestamps[5] = 1_000_004_687L; // DoD=-237 (bucket 2) - timestamps[6] = 1_000_006_387L; // DoD=1000 (bucket 3) - timestamps[7] = 1_000_020_087L; // DoD=12000 (bucket 4) - timestamps[8] = 1_000_033_787L; // DoD=0 (bucket 0) - timestamps[9] = 1_000_047_487L; // DoD=0 (bucket 0) - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testEncodeTimestampsRoundTripConstantDelta() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); + public void testEncodeTimestampsRoundTripConstantDelta() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } - long[] timestamps = new long[100]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = 1_000_000_000L + i * 1000L; - } + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testEncodeTimestampsRoundTripLargeDataset() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - int count = 10_000; - long[] timestamps = new long[count]; - timestamps[0] = 1_000_000_000L; - timestamps[1] = 1_000_001_000L; - - java.util.Random random = new java.util.Random(42); - for (int i = 2; i < count; i++) { - long prevDelta = timestamps[i - 1] - timestamps[i - 2]; - int variation = (i % 10 == 0) ? random.nextInt(100) - 50 : 0; - timestamps[i] = timestamps[i - 1] + prevDelta + variation; - } + public void testEncodeTimestampsRoundTripLargeDataset() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + int count = 10_000; + long[] timestamps = new long[count]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, count) + 100; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, count); - Assert.assertTrue(bytesWritten > 0); - Assert.assertTrue("Should compress better than uncompressed", bytesWritten < count * 8); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; + java.util.Random random = new java.util.Random(42); for (int i = 2; i < count; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; + long prevDelta = timestamps[i - 1] - timestamps[i - 2]; + int variation = (i % 10 == 0) ? random.nextInt(100) - 50 : 0; + timestamps[i] = timestamps[i - 1] + prevDelta + variation; } - } finally { - Unsafe.free(src, (long) count * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, count) + 100; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, count); + Assert.assertTrue(bytesWritten > 0); + Assert.assertTrue("Should compress better than uncompressed", bytesWritten < count * 8); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < count; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) count * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testEncodeTimestampsRoundTripNegativeDoD() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - long[] timestamps = { - 1_000_000_000L, - 1_000_002_000L, // delta=2000 - 1_000_003_000L, // DoD=-1000 (bucket 3) - 1_000_003_500L, // DoD=-500 (bucket 2) - 1_000_003_600L, // DoD=-400 (bucket 2) - }; - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; + public void testEncodeTimestampsRoundTripNegativeDoD() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = { + 1_000_000_000L, + 1_000_002_000L, // delta=2000 + 1_000_003_000L, // DoD=-1000 (bucket 3) + 1_000_003_500L, // DoD=-500 (bucket 2) + 1_000_003_600L, // DoD=-400 (bucket 2) + }; + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testEncodeTimestampsRoundTripVaryingDelta() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - long[] timestamps = { - 1_000_000_000L, - 1_000_001_000L, // delta=1000 - 1_000_002_000L, // DoD=0 (bucket 0) - 1_000_003_010L, // DoD=10 (bucket 1) - 1_000_004_120L, // DoD=100 (bucket 1) - 1_000_005_420L, // DoD=190 (bucket 2) - 1_000_007_720L, // DoD=1000 (bucket 3) - 1_000_020_020L, // DoD=10000 (bucket 4) - }; - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; + public void testEncodeTimestampsRoundTripVaryingDelta() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = { + 1_000_000_000L, + 1_000_001_000L, // delta=1000 + 1_000_002_000L, // DoD=0 (bucket 0) + 1_000_003_010L, // DoD=10 (bucket 1) + 1_000_004_120L, // DoD=100 (bucket 1) + 1_000_005_420L, // DoD=190 (bucket 2) + 1_000_007_720L, // DoD=1000 (bucket 3) + 1_000_020_020L, // DoD=10000 (bucket 4) + }; + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testEncodeTimestampsTwoTimestamps() { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - long[] timestamps = {1_000_000_000L, 1_000_001_000L}; - long src = putTimestamps(timestamps); - long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 2); - Assert.assertEquals(16, bytesWritten); - Assert.assertEquals(1_000_000_000L, Unsafe.getUnsafe().getLong(dst)); - Assert.assertEquals(1_000_001_000L, Unsafe.getUnsafe().getLong(dst + 8)); - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); - } + public void testEncodeTimestampsTwoTimestamps() throws Exception { + assertMemoryLeak(() -> { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long[] timestamps = {1_000_000_000L, 1_000_001_000L}; + long src = putTimestamps(timestamps); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 2); + Assert.assertEquals(16, bytesWritten); + Assert.assertEquals(1_000_000_000L, Unsafe.getUnsafe().getLong(dst)); + Assert.assertEquals(1_000_001_000L, Unsafe.getUnsafe().getLong(dst + 8)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testGetBitsRequired() { - // Bucket 0: 1 bit - Assert.assertEquals(1, QwpGorillaEncoder.getBitsRequired(0)); + public void testGetBitsRequired() throws Exception { + assertMemoryLeak(() -> { + // Bucket 0: 1 bit + Assert.assertEquals(1, QwpGorillaEncoder.getBitsRequired(0)); - // Bucket 1: 9 bits (2 prefix + 7 value) - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(1)); - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-1)); - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(63)); - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-64)); + // Bucket 1: 9 bits (2 prefix + 7 value) + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(1)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-1)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(63)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-64)); - // Bucket 2: 12 bits (3 prefix + 9 value) - Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(64)); - Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(255)); - Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(-256)); + // Bucket 2: 12 bits (3 prefix + 9 value) + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(64)); + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(255)); + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(-256)); - // Bucket 3: 16 bits (4 prefix + 12 value) - Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(256)); - Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(2047)); - Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(-2048)); + // Bucket 3: 16 bits (4 prefix + 12 value) + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(256)); + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(2047)); + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(-2048)); - // Bucket 4: 36 bits (4 prefix + 32 value) - Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(2048)); - Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(-2049)); + // Bucket 4: 36 bits (4 prefix + 32 value) + Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(2048)); + Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(-2049)); + }); } @Test - public void testGetBucket12Bit() { - // DoD in [-2048, 2047] but outside [-256, 255] -> bucket 3 (16 bits) - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(256)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-257)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(2047)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2047)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2048)); + public void testGetBucket12Bit() throws Exception { + assertMemoryLeak(() -> { + // DoD in [-2048, 2047] but outside [-256, 255] -> bucket 3 (16 bits) + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(256)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-257)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(2047)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2047)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2048)); + }); } @Test - public void testGetBucket32Bit() { - // DoD outside [-2048, 2047] -> bucket 4 (36 bits) - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(2048)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-2049)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(100_000)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-100_000)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MAX_VALUE)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MIN_VALUE)); + public void testGetBucket32Bit() throws Exception { + assertMemoryLeak(() -> { + // DoD outside [-2048, 2047] -> bucket 4 (36 bits) + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(2048)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-2049)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(100_000)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-100_000)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MAX_VALUE)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MIN_VALUE)); + }); } @Test - public void testGetBucket7Bit() { - // DoD in [-64, 63] -> bucket 1 (9 bits) - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(1)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-1)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(63)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-63)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-64)); + public void testGetBucket7Bit() throws Exception { + assertMemoryLeak(() -> { + // DoD in [-64, 63] -> bucket 1 (9 bits) + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(1)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-1)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(63)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-63)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-64)); + }); } @Test - public void testGetBucket9Bit() { - // DoD in [-256, 255] but outside [-64, 63] -> bucket 2 (12 bits) - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(64)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-65)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(255)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-255)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-256)); + public void testGetBucket9Bit() throws Exception { + assertMemoryLeak(() -> { + // DoD in [-256, 255] but outside [-64, 63] -> bucket 2 (12 bits) + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(64)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-65)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(255)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-255)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-256)); + }); } @Test - public void testGetBucketZero() { - Assert.assertEquals(0, QwpGorillaEncoder.getBucket(0)); + public void testGetBucketZero() throws Exception { + assertMemoryLeak(() -> { + Assert.assertEquals(0, QwpGorillaEncoder.getBucket(0)); + }); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java index 086d24f..2a7491d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java @@ -27,263 +27,290 @@ import io.questdb.client.cutlass.qwp.protocol.QwpNullBitmap; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; public class QwpNullBitmapTest { @Test - public void testAllNulls() { - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillAllNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); - Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); - - for (int i = 0; i < rowCount; i++) { - Assert.assertTrue(QwpNullBitmap.isNull(address, i)); + public void testAllNulls() throws Exception { + assertMemoryLeak(() -> { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); + + for (int i = 0; i < rowCount; i++) { + Assert.assertTrue(QwpNullBitmap.isNull(address, i)); + } + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testAllNullsPartialByte() { - // Test with row count not divisible by 8 - int rowCount = 10; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillAllNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); - Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + public void testAllNullsPartialByte() throws Exception { + assertMemoryLeak(() -> { + // Test with row count not divisible by 8 + int rowCount = 10; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testBitmapBitOrder() { - // Test LSB-first bit ordering - int rowCount = 8; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set bit 0 (LSB) - QwpNullBitmap.setNull(address, 0); - byte b = Unsafe.getUnsafe().getByte(address); - Assert.assertEquals(0b00000001, b & 0xFF); - - // Set bit 7 (MSB of first byte) - QwpNullBitmap.setNull(address, 7); - b = Unsafe.getUnsafe().getByte(address); - Assert.assertEquals(0b10000001, b & 0xFF); - - // Set bit 3 - QwpNullBitmap.setNull(address, 3); - b = Unsafe.getUnsafe().getByte(address); - Assert.assertEquals(0b10001001, b & 0xFF); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + public void testBitmapBitOrder() throws Exception { + assertMemoryLeak(() -> { + // Test LSB-first bit ordering + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set bit 0 (LSB) + QwpNullBitmap.setNull(address, 0); + byte b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b00000001, b & 0xFF); + + // Set bit 7 (MSB of first byte) + QwpNullBitmap.setNull(address, 7); + b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b10000001, b & 0xFF); + + // Set bit 3 + QwpNullBitmap.setNull(address, 3); + b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b10001001, b & 0xFF); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testBitmapByteAlignment() { - // Test that bits 8-15 go into second byte - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set bit 8 (first bit of second byte) - QwpNullBitmap.setNull(address, 8); - Assert.assertEquals(0, Unsafe.getUnsafe().getByte(address) & 0xFF); - Assert.assertEquals(0b00000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); - - // Set bit 15 (last bit of second byte) - QwpNullBitmap.setNull(address, 15); - Assert.assertEquals(0b10000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + public void testBitmapByteAlignment() throws Exception { + assertMemoryLeak(() -> { + // Test that bits 8-15 go into second byte + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set bit 8 (first bit of second byte) + QwpNullBitmap.setNull(address, 8); + Assert.assertEquals(0, Unsafe.getUnsafe().getByte(address) & 0xFF); + Assert.assertEquals(0b00000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + + // Set bit 15 (last bit of second byte) + QwpNullBitmap.setNull(address, 15); + Assert.assertEquals(0b10000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testBitmapSizeCalculation() { - Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); - Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(1)); - Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(7)); - Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(8)); - Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(9)); - Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(16)); - Assert.assertEquals(3, QwpNullBitmap.sizeInBytes(17)); - Assert.assertEquals(125, QwpNullBitmap.sizeInBytes(1000)); - Assert.assertEquals(125000, QwpNullBitmap.sizeInBytes(1000000)); + public void testBitmapSizeCalculation() throws Exception { + assertMemoryLeak(() -> { + Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(1)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(7)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(8)); + Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(9)); + Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(16)); + Assert.assertEquals(3, QwpNullBitmap.sizeInBytes(17)); + Assert.assertEquals(125, QwpNullBitmap.sizeInBytes(1000)); + Assert.assertEquals(125000, QwpNullBitmap.sizeInBytes(1000000)); + }); } @Test - public void testBitmapWithPartialLastByte() { - // 10 rows = 2 bytes, but only 2 bits used in second byte - int rowCount = 10; - int size = QwpNullBitmap.sizeInBytes(rowCount); - Assert.assertEquals(2, size); - - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set row 9 (bit 1 of second byte) - QwpNullBitmap.setNull(address, 9); - Assert.assertTrue(QwpNullBitmap.isNull(address, 9)); - Assert.assertEquals(0b00000010, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); - - Assert.assertEquals(1, QwpNullBitmap.countNulls(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + public void testBitmapWithPartialLastByte() throws Exception { + assertMemoryLeak(() -> { + // 10 rows = 2 bytes, but only 2 bits used in second byte + int rowCount = 10; + int size = QwpNullBitmap.sizeInBytes(rowCount); + Assert.assertEquals(2, size); + + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set row 9 (bit 1 of second byte) + QwpNullBitmap.setNull(address, 9); + Assert.assertTrue(QwpNullBitmap.isNull(address, 9)); + Assert.assertEquals(0b00000010, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + + Assert.assertEquals(1, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testByteArrayOperations() { - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - byte[] bitmap = new byte[size]; - int offset = 0; - - QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); - - QwpNullBitmap.setNull(bitmap, offset, 0); - QwpNullBitmap.setNull(bitmap, offset, 5); - QwpNullBitmap.setNull(bitmap, offset, 15); - - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 0)); - Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 1)); - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 5)); - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 15)); - - Assert.assertEquals(3, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); - - QwpNullBitmap.clearNull(bitmap, offset, 5); - Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 5)); - Assert.assertEquals(2, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); + public void testByteArrayOperations() throws Exception { + assertMemoryLeak(() -> { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + byte[] bitmap = new byte[size]; + int offset = 0; + + QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); + + QwpNullBitmap.setNull(bitmap, offset, 0); + QwpNullBitmap.setNull(bitmap, offset, 5); + QwpNullBitmap.setNull(bitmap, offset, 15); + + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 0)); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 1)); + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 5)); + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 15)); + + Assert.assertEquals(3, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); + + QwpNullBitmap.clearNull(bitmap, offset, 5); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 5)); + Assert.assertEquals(2, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); + }); } @Test - public void testByteArrayWithOffset() { - int rowCount = 8; - int size = QwpNullBitmap.sizeInBytes(rowCount); - byte[] bitmap = new byte[10 + size]; // Extra padding - int offset = 5; // Start at offset 5 - - QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); - QwpNullBitmap.setNull(bitmap, offset, 3); - - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 3)); - Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 4)); + public void testByteArrayWithOffset() throws Exception { + assertMemoryLeak(() -> { + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + byte[] bitmap = new byte[10 + size]; // Extra padding + int offset = 5; // Start at offset 5 + + QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); + QwpNullBitmap.setNull(bitmap, offset, 3); + + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 3)); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 4)); + }); } @Test - public void testClearNull() { - int rowCount = 8; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillAllNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.isNull(address, 3)); - - QwpNullBitmap.clearNull(address, 3); - Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); - Assert.assertEquals(7, QwpNullBitmap.countNulls(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + public void testClearNull() throws Exception { + assertMemoryLeak(() -> { + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.isNull(address, 3)); + + QwpNullBitmap.clearNull(address, 3); + Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); + Assert.assertEquals(7, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testEmptyBitmap() { - Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); + public void testEmptyBitmap() throws Exception { + assertMemoryLeak(() -> { + Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); + }); } @Test - public void testLargeBitmap() { - int rowCount = 100000; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set every 100th row as null - int expectedNulls = 0; - for (int i = 0; i < rowCount; i += 100) { - QwpNullBitmap.setNull(address, i); - expectedNulls++; + public void testLargeBitmap() throws Exception { + assertMemoryLeak(() -> { + int rowCount = 100000; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set every 100th row as null + int expectedNulls = 0; + for (int i = 0; i < rowCount; i += 100) { + QwpNullBitmap.setNull(address, i); + expectedNulls++; + } + + Assert.assertEquals(expectedNulls, QwpNullBitmap.countNulls(address, rowCount)); + + // Verify some random positions + Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 100)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 99900)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 99)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); } - - Assert.assertEquals(expectedNulls, QwpNullBitmap.countNulls(address, rowCount)); - - // Verify some random positions - Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 100)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 99900)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 99)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test - public void testMixedNulls() { - int rowCount = 20; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set specific rows as null: 0, 2, 5, 19 - QwpNullBitmap.setNull(address, 0); - QwpNullBitmap.setNull(address, 2); - QwpNullBitmap.setNull(address, 5); - QwpNullBitmap.setNull(address, 19); - - Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 2)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 4)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 5)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 19)); - - Assert.assertEquals(4, QwpNullBitmap.countNulls(address, rowCount)); - Assert.assertFalse(QwpNullBitmap.allNull(address, rowCount)); - Assert.assertFalse(QwpNullBitmap.noneNull(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + public void testMixedNulls() throws Exception { + assertMemoryLeak(() -> { + int rowCount = 20; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set specific rows as null: 0, 2, 5, 19 + QwpNullBitmap.setNull(address, 0); + QwpNullBitmap.setNull(address, 2); + QwpNullBitmap.setNull(address, 5); + QwpNullBitmap.setNull(address, 19); + + Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 2)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 4)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 5)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 19)); + + Assert.assertEquals(4, QwpNullBitmap.countNulls(address, rowCount)); + Assert.assertFalse(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertFalse(QwpNullBitmap.noneNull(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test - public void testNoNulls() { - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.noneNull(address, rowCount)); - Assert.assertEquals(0, QwpNullBitmap.countNulls(address, rowCount)); - - for (int i = 0; i < rowCount; i++) { - Assert.assertFalse(QwpNullBitmap.isNull(address, i)); + public void testNoNulls() throws Exception { + assertMemoryLeak(() -> { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.noneNull(address, rowCount)); + Assert.assertEquals(0, QwpNullBitmap.countNulls(address, rowCount)); + + for (int i = 0; i < rowCount; i++) { + Assert.assertFalse(QwpNullBitmap.isNull(address, i)); + } + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); } - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } + }); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java index 75cae68..129f56b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.protocol.QwpSchemaHash; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -238,21 +239,23 @@ public void testTypeAffectsHash() { } @Test - public void testXXHash64DirectMemory() { - byte[] data = "test data".getBytes(StandardCharsets.UTF_8); - long addr = Unsafe.malloc(data.length, MemoryTag.NATIVE_ILP_RSS); - try { - for (int i = 0; i < data.length; i++) { - Unsafe.getUnsafe().putByte(addr + i, data[i]); + public void testXXHash64DirectMemory() throws Exception { + assertMemoryLeak(() -> { + byte[] data = "test data".getBytes(StandardCharsets.UTF_8); + long addr = Unsafe.malloc(data.length, MemoryTag.NATIVE_ILP_RSS); + try { + for (int i = 0; i < data.length; i++) { + Unsafe.getUnsafe().putByte(addr + i, data[i]); + } + + long hashFromBytes = QwpSchemaHash.hash(data); + long hashFromMem = QwpSchemaHash.hash(addr, data.length); + + Assert.assertEquals("Direct memory hash should match byte array hash", hashFromBytes, hashFromMem); + } finally { + Unsafe.free(addr, data.length, MemoryTag.NATIVE_ILP_RSS); } - - long hashFromBytes = QwpSchemaHash.hash(data); - long hashFromMem = QwpSchemaHash.hash(addr, data.length); - - Assert.assertEquals("Direct memory hash should match byte array hash", hashFromBytes, hashFromMem); - } finally { - Unsafe.free(addr, data.length, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 2ed4762..47806b0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -30,6 +30,7 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal64; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static org.junit.Assert.assertArrayEquals; @@ -39,400 +40,426 @@ public class QwpTableBufferTest { @Test - public void testAddDecimal128RescaleOverflow() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL128, true); - // First row sets decimalScale = 10 - col.addDecimal128(Decimal128.fromLong(1, 10)); - table.nextRow(); - // Second row at scale 0 with a large value — rescaling to scale 10 - // multiplies by 10^10, which exceeds 128-bit capacity - try { - col.addDecimal128(new Decimal128(Long.MAX_VALUE / 2, Long.MAX_VALUE, 0)); - fail("Expected LineSenderException for 128-bit overflow"); - } catch (LineSenderException e) { - assertEquals("Decimal128 overflow: rescaling from scale 0 to 10 exceeds 128-bit capacity", e.getMessage()); + public void testAddDecimal128RescaleOverflow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL128, true); + // First row sets decimalScale = 10 + col.addDecimal128(Decimal128.fromLong(1, 10)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 10 + // multiplies by 10^10, which exceeds 128-bit capacity + try { + col.addDecimal128(new Decimal128(Long.MAX_VALUE / 2, Long.MAX_VALUE, 0)); + fail("Expected LineSenderException for 128-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal128 overflow: rescaling from scale 0 to 10 exceeds 128-bit capacity", e.getMessage()); + } } - } + }); } @Test - public void testAddDecimal64RescaleOverflow() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); - // First row sets decimalScale = 5 - col.addDecimal64(Decimal64.fromLong(1, 5)); - table.nextRow(); - // Second row at scale 0 with a large value — rescaling to scale 5 - // multiplies by 10^5 = 100_000, which exceeds 64-bit capacity - // Long.MAX_VALUE / 10 ≈ 9.2 * 10^17, * 10^5 ≈ 9.2 * 10^22 >> 2^63 - try { - col.addDecimal64(Decimal64.fromLong(Long.MAX_VALUE / 10, 0)); - fail("Expected LineSenderException for 64-bit overflow"); - } catch (LineSenderException e) { - assertEquals("Decimal64 overflow: rescaling from scale 0 to 5 exceeds 64-bit capacity", e.getMessage()); + public void testAddDecimal64RescaleOverflow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + // First row sets decimalScale = 5 + col.addDecimal64(Decimal64.fromLong(1, 5)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 5 + // multiplies by 10^5 = 100_000, which exceeds 64-bit capacity + // Long.MAX_VALUE / 10 ≈ 9.2 * 10^17, * 10^5 ≈ 9.2 * 10^22 >> 2^63 + try { + col.addDecimal64(Decimal64.fromLong(Long.MAX_VALUE / 10, 0)); + fail("Expected LineSenderException for 64-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal64 overflow: rescaling from scale 0 to 5 exceeds 64-bit capacity", e.getMessage()); + } } - } + }); } @Test - public void testAddDoubleArrayNullOnNonNullableColumn() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - - // Row 0: real array - col.addDoubleArray(new double[]{1.0, 2.0}); - table.nextRow(); - - // Row 1: null on non-nullable — must write empty array metadata - col.addDoubleArray((double[]) null); - table.nextRow(); - - // Row 2: real array - col.addDoubleArray(new double[]{3.0, 4.0}); - table.nextRow(); - - assertEquals(3, table.getRowCount()); - assertEquals(3, col.getValueCount()); - assertEquals(col.getSize(), col.getValueCount()); - - // Encoder walk must not corrupt — row 1 is an empty array - double[] encoded = readDoubleArraysLikeEncoder(col); - assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, encoded, 0.0); - - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - assertEquals(1, dims[0]); - assertEquals(2, shapes[0]); - assertEquals(1, dims[1]); // null row: 1D empty - assertEquals(0, shapes[1]); // null row: 0 elements - assertEquals(1, dims[2]); - assertEquals(2, shapes[2]); - } - } + public void testAddDoubleArrayNullOnNonNullableColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - @Test - public void testAddLongArrayNullOnNonNullableColumn() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - - // Row 0: real array - col.addLongArray(new long[]{10, 20}); - table.nextRow(); - - // Row 1: null on non-nullable — must write empty array metadata - col.addLongArray((long[]) null); - table.nextRow(); - - // Row 2: real array - col.addLongArray(new long[]{30, 40}); - table.nextRow(); - - assertEquals(3, table.getRowCount()); - assertEquals(3, col.getValueCount()); - assertEquals(col.getSize(), col.getValueCount()); - - // Encoder walk must not corrupt — row 1 is an empty array - long[] encoded = readLongArraysLikeEncoder(col); - assertArrayEquals(new long[]{10, 20, 30, 40}, encoded); - - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - assertEquals(1, dims[0]); - assertEquals(2, shapes[0]); - assertEquals(1, dims[1]); // null row: 1D empty - assertEquals(0, shapes[1]); // null row: 0 elements - assertEquals(1, dims[2]); - assertEquals(2, shapes[2]); - } - } + // Row 0: real array + col.addDoubleArray(new double[]{1.0, 2.0}); + table.nextRow(); - @Test - public void testAddSymbolNullOnNonNullableColumn() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, false); - col.addSymbol("server1"); - table.nextRow(); - - // Null on a non-nullable column must write a sentinel value, - // keeping size and valueCount in sync - col.addSymbol(null); - table.nextRow(); - - col.addSymbol("server2"); - table.nextRow(); - - assertEquals(3, table.getRowCount()); - // For non-nullable columns, every row must have a physical value - assertEquals(col.getSize(), col.getValueCount()); - } + // Row 1: null on non-nullable — must write empty array metadata + col.addDoubleArray((double[]) null); + table.nextRow(); + + // Row 2: real array + col.addDoubleArray(new double[]{3.0, 4.0}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, encoded, 0.0); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + }); } @Test - public void testCancelRowRewindsDoubleArrayOffsets() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - // Row 0: committed with [1.0, 2.0] - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{1.0, 2.0}); - table.nextRow(); - - // Row 1: committed with [3.0, 4.0] - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{3.0, 4.0}); - table.nextRow(); - - // Start row 2 with [5.0, 6.0] — then cancel it - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{5.0, 6.0}); - table.cancelCurrentRow(); - - // Add replacement row 2 with [7.0, 8.0] - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{7.0, 8.0}); - table.nextRow(); - - assertEquals(3, table.getRowCount()); - assertEquals(3, col.getValueCount()); - - // Walk the arrays exactly as the encoder would - double[] encoded = readDoubleArraysLikeEncoder(col); - assertArrayEquals( - new double[]{1.0, 2.0, 3.0, 4.0, 7.0, 8.0}, - encoded, - 0.0 - ); - } + public void testAddLongArrayNullOnNonNullableColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + // Row 0: real array + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + // Row 1: null on non-nullable — must write empty array metadata + col.addLongArray((long[]) null); + table.nextRow(); + + // Row 2: real array + col.addLongArray(new long[]{30, 40}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 30, 40}, encoded); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + }); } @Test - public void testCancelRowRewindsLongArrayOffsets() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - // Row 0: committed with [10, 20] - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - col.addLongArray(new long[]{10, 20}); - table.nextRow(); - - // Start row 1 with [30, 40] — then cancel it - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - col.addLongArray(new long[]{30, 40}); - table.cancelCurrentRow(); - - // Add replacement row 1 with [50, 60] - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - col.addLongArray(new long[]{50, 60}); - table.nextRow(); - - assertEquals(2, table.getRowCount()); - assertEquals(2, col.getValueCount()); - - long[] encoded = readLongArraysLikeEncoder(col); - assertArrayEquals(new long[]{10, 20, 50, 60}, encoded); - } + public void testAddSymbolNullOnNonNullableColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, false); + col.addSymbol("server1"); + table.nextRow(); + + // Null on a non-nullable column must write a sentinel value, + // keeping size and valueCount in sync + col.addSymbol(null); + table.nextRow(); + + col.addSymbol("server2"); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + // For non-nullable columns, every row must have a physical value + assertEquals(col.getSize(), col.getValueCount()); + } + }); } @Test - public void testCancelRowRewindsMultiDimArrayOffsets() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - // Row 0: committed 2D array [[1.0, 2.0], [3.0, 4.0]] - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); - table.nextRow(); - - // Start row 1 with 2D array [[5.0, 6.0], [7.0, 8.0]] — cancel - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[][]{{5.0, 6.0}, {7.0, 8.0}}); - table.cancelCurrentRow(); - - // Replacement row 1 with [[9.0, 10.0], [11.0, 12.0]] - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[][]{{9.0, 10.0}, {11.0, 12.0}}); - table.nextRow(); - - assertEquals(2, table.getRowCount()); - assertEquals(2, col.getValueCount()); - - // Verify shapes are correct (2 dims per row, each [2, 2]) - int[] shapes = col.getArrayShapes(); - byte[] dims = col.getArrayDims(); - assertEquals(2, dims[0]); - assertEquals(2, dims[1]); - // Row 0 shapes: [2, 2] - assertEquals(2, shapes[0]); - assertEquals(2, shapes[1]); - // Row 1 shapes must be the replacement [2, 2], not stale data - assertEquals(2, shapes[2]); - assertEquals(2, shapes[3]); - - double[] encoded = readDoubleArraysLikeEncoder(col); - assertArrayEquals( - new double[]{1.0, 2.0, 3.0, 4.0, 9.0, 10.0, 11.0, 12.0}, - encoded, - 0.0 - ); - } + public void testCancelRowRewindsDoubleArrayOffsets() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Row 0: committed with [1.0, 2.0] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{1.0, 2.0}); + table.nextRow(); + + // Row 1: committed with [3.0, 4.0] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{3.0, 4.0}); + table.nextRow(); + + // Start row 2 with [5.0, 6.0] — then cancel it + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{5.0, 6.0}); + table.cancelCurrentRow(); + + // Add replacement row 2 with [7.0, 8.0] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{7.0, 8.0}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + + // Walk the arrays exactly as the encoder would + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 7.0, 8.0}, + encoded, + 0.0 + ); + } + }); } @Test - public void testDoubleArrayWrapperMultipleRows() { - try (QwpTableBuffer table = new QwpTableBuffer("test"); - DoubleArray arr = new DoubleArray(3)) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - - arr.append(1.0).append(2.0).append(3.0); - col.addDoubleArray(arr); - table.nextRow(); - - // DoubleArray auto-wraps, so just append next row's data - arr.append(4.0).append(5.0).append(6.0); - col.addDoubleArray(arr); - table.nextRow(); - - arr.append(7.0).append(8.0).append(9.0); - col.addDoubleArray(arr); - table.nextRow(); - - assertEquals(3, col.getValueCount()); - double[] encoded = readDoubleArraysLikeEncoder(col); - assertArrayEquals( - new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0}, - encoded, - 0.0 - ); - - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - for (int i = 0; i < 3; i++) { - assertEquals(1, dims[i]); - assertEquals(3, shapes[i]); + public void testCancelRowRewindsLongArrayOffsets() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Row 0: committed with [10, 20] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + // Start row 1 with [30, 40] — then cancel it + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{30, 40}); + table.cancelCurrentRow(); + + // Add replacement row 1 with [50, 60] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{50, 60}); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, col.getValueCount()); + + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 50, 60}, encoded); } - } + }); } @Test - public void testDoubleArrayWrapperShrinkingSize() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - - // Row 0: large array (5 elements) - try (DoubleArray big = new DoubleArray(5)) { - big.append(1.0).append(2.0).append(3.0).append(4.0).append(5.0); - col.addDoubleArray(big); + public void testCancelRowRewindsMultiDimArrayOffsets() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Row 0: committed 2D array [[1.0, 2.0], [3.0, 4.0]] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); table.nextRow(); - } - // Row 1: smaller array (2 elements) — must not see leftover data from row 0 - try (DoubleArray small = new DoubleArray(2)) { - small.append(10.0).append(20.0); - col.addDoubleArray(small); + // Start row 1 with 2D array [[5.0, 6.0], [7.0, 8.0]] — cancel + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[][]{{5.0, 6.0}, {7.0, 8.0}}); + table.cancelCurrentRow(); + + // Replacement row 1 with [[9.0, 10.0], [11.0, 12.0]] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[][]{{9.0, 10.0}, {11.0, 12.0}}); table.nextRow(); - } - assertEquals(2, col.getValueCount()); - double[] encoded = readDoubleArraysLikeEncoder(col); - assertArrayEquals( - new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0}, - encoded, - 0.0 - ); - - int[] shapes = col.getArrayShapes(); - assertEquals(5, shapes[0]); - assertEquals(2, shapes[1]); - } + assertEquals(2, table.getRowCount()); + assertEquals(2, col.getValueCount()); + + // Verify shapes are correct (2 dims per row, each [2, 2]) + int[] shapes = col.getArrayShapes(); + byte[] dims = col.getArrayDims(); + assertEquals(2, dims[0]); + assertEquals(2, dims[1]); + // Row 0 shapes: [2, 2] + assertEquals(2, shapes[0]); + assertEquals(2, shapes[1]); + // Row 1 shapes must be the replacement [2, 2], not stale data + assertEquals(2, shapes[2]); + assertEquals(2, shapes[3]); + + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 9.0, 10.0, 11.0, 12.0}, + encoded, + 0.0 + ); + } + }); } @Test - public void testDoubleArrayWrapperVaryingDimensionality() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - - // Row 0: 2D array (2x2) - try (DoubleArray matrix = new DoubleArray(2, 2)) { - matrix.append(1.0).append(2.0).append(3.0).append(4.0); - col.addDoubleArray(matrix); + public void testDoubleArrayWrapperMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test"); + DoubleArray arr = new DoubleArray(3)) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + arr.append(1.0).append(2.0).append(3.0); + col.addDoubleArray(arr); table.nextRow(); - } - // Row 1: 1D array (3 elements) — different dimensionality - try (DoubleArray vec = new DoubleArray(3)) { - vec.append(10.0).append(20.0).append(30.0); - col.addDoubleArray(vec); + // DoubleArray auto-wraps, so just append next row's data + arr.append(4.0).append(5.0).append(6.0); + col.addDoubleArray(arr); table.nextRow(); + + arr.append(7.0).append(8.0).append(9.0); + col.addDoubleArray(arr); + table.nextRow(); + + assertEquals(3, col.getValueCount()); + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0}, + encoded, + 0.0 + ); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + for (int i = 0; i < 3; i++) { + assertEquals(1, dims[i]); + assertEquals(3, shapes[i]); + } } + }); + } - assertEquals(2, col.getValueCount()); - - byte[] dims = col.getArrayDims(); - assertEquals(2, dims[0]); - assertEquals(1, dims[1]); - - int[] shapes = col.getArrayShapes(); - // Row 0: shape [2, 2] - assertEquals(2, shapes[0]); - assertEquals(2, shapes[1]); - // Row 1: shape [3] - assertEquals(3, shapes[2]); - - double[] encoded = readDoubleArraysLikeEncoder(col); - assertArrayEquals( - new double[]{1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 30.0}, - encoded, - 0.0 - ); - } + @Test + public void testDoubleArrayWrapperShrinkingSize() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + // Row 0: large array (5 elements) + try (DoubleArray big = new DoubleArray(5)) { + big.append(1.0).append(2.0).append(3.0).append(4.0).append(5.0); + col.addDoubleArray(big); + table.nextRow(); + } + + // Row 1: smaller array (2 elements) — must not see leftover data from row 0 + try (DoubleArray small = new DoubleArray(2)) { + small.append(10.0).append(20.0); + col.addDoubleArray(small); + table.nextRow(); + } + + assertEquals(2, col.getValueCount()); + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0}, + encoded, + 0.0 + ); + + int[] shapes = col.getArrayShapes(); + assertEquals(5, shapes[0]); + assertEquals(2, shapes[1]); + } + }); } @Test - public void testLongArrayMultipleRows() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - - col.addLongArray(new long[]{10, 20, 30}); - table.nextRow(); - - col.addLongArray(new long[]{40, 50, 60}); - table.nextRow(); - - col.addLongArray(new long[]{70, 80, 90}); - table.nextRow(); - - assertEquals(3, col.getValueCount()); - long[] encoded = readLongArraysLikeEncoder(col); - assertArrayEquals( - new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90}, - encoded - ); - - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - for (int i = 0; i < 3; i++) { - assertEquals(1, dims[i]); - assertEquals(3, shapes[i]); + public void testDoubleArrayWrapperVaryingDimensionality() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + // Row 0: 2D array (2x2) + try (DoubleArray matrix = new DoubleArray(2, 2)) { + matrix.append(1.0).append(2.0).append(3.0).append(4.0); + col.addDoubleArray(matrix); + table.nextRow(); + } + + // Row 1: 1D array (3 elements) — different dimensionality + try (DoubleArray vec = new DoubleArray(3)) { + vec.append(10.0).append(20.0).append(30.0); + col.addDoubleArray(vec); + table.nextRow(); + } + + assertEquals(2, col.getValueCount()); + + byte[] dims = col.getArrayDims(); + assertEquals(2, dims[0]); + assertEquals(1, dims[1]); + + int[] shapes = col.getArrayShapes(); + // Row 0: shape [2, 2] + assertEquals(2, shapes[0]); + assertEquals(2, shapes[1]); + // Row 1: shape [3] + assertEquals(3, shapes[2]); + + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 30.0}, + encoded, + 0.0 + ); } - } + }); } @Test - public void testLongArrayShrinkingSize() { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + public void testLongArrayMultipleRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - // Row 0: large array (4 elements) - col.addLongArray(new long[]{100, 200, 300, 400}); - table.nextRow(); + col.addLongArray(new long[]{10, 20, 30}); + table.nextRow(); - // Row 1: smaller array (2 elements) — must not see leftover data from row 0 - col.addLongArray(new long[]{10, 20}); - table.nextRow(); + col.addLongArray(new long[]{40, 50, 60}); + table.nextRow(); - assertEquals(2, col.getValueCount()); - long[] encoded = readLongArraysLikeEncoder(col); - assertArrayEquals(new long[]{100, 200, 300, 400, 10, 20}, encoded); + col.addLongArray(new long[]{70, 80, 90}); + table.nextRow(); - int[] shapes = col.getArrayShapes(); - assertEquals(4, shapes[0]); - assertEquals(2, shapes[1]); - } + assertEquals(3, col.getValueCount()); + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals( + new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90}, + encoded + ); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + for (int i = 0; i < 3; i++) { + assertEquals(1, dims[i]); + assertEquals(3, shapes[i]); + } + } + }); + } + + @Test + public void testLongArrayShrinkingSize() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + // Row 0: large array (4 elements) + col.addLongArray(new long[]{100, 200, 300, 400}); + table.nextRow(); + + // Row 1: smaller array (2 elements) — must not see leftover data from row 0 + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + assertEquals(2, col.getValueCount()); + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{100, 200, 300, 400, 10, 20}, encoded); + + int[] shapes = col.getArrayShapes(); + assertEquals(4, shapes[0]); + assertEquals(2, shapes[1]); + } + }); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java index 558ecd2..aae8ed1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.protocol.QwpVarint; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -35,22 +36,24 @@ public class QwpVarintTest { @Test - public void testDecodeFromDirectMemory() { - long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - try { - // Encode using byte array, decode from direct memory - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 300); - - for (int i = 0; i < len; i++) { - Unsafe.getUnsafe().putByte(addr + i, buf[i]); + public void testDecodeFromDirectMemory() throws Exception { + assertMemoryLeak(() -> { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + // Encode using byte array, decode from direct memory + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 300); + + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(addr + i, buf[i]); + } + + long decoded = QwpVarint.decode(addr, addr + len); + Assert.assertEquals(300, decoded); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); } - - long decoded = QwpVarint.decode(addr, addr + len); - Assert.assertEquals(300, decoded); - } finally { - Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); - } + }); } @Test @@ -95,20 +98,22 @@ public void testDecodeResult() { } @Test - public void testDecodeResultFromDirectMemory() { - long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - try { - long endAddr = QwpVarint.encode(addr, 999999); - int expectedLen = (int) (endAddr - addr); - - QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); - QwpVarint.decode(addr, endAddr, result); - - Assert.assertEquals(999999, result.value); - Assert.assertEquals(expectedLen, result.bytesRead); - } finally { - Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); - } + public void testDecodeResultFromDirectMemory() throws Exception { + assertMemoryLeak(() -> { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + long endAddr = QwpVarint.encode(addr, 999999); + int expectedLen = (int) (endAddr - addr); + + QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); + QwpVarint.decode(addr, endAddr, result); + + Assert.assertEquals(999999, result.value); + Assert.assertEquals(expectedLen, result.bytesRead); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test @@ -216,19 +221,21 @@ public void testEncodeSpecificValues() { } @Test - public void testEncodeToDirectMemory() { - long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - try { - long endAddr = QwpVarint.encode(addr, 12345); - int len = (int) (endAddr - addr); - Assert.assertTrue(len > 0); - - // Read back and verify - long decoded = QwpVarint.decode(addr, endAddr); - Assert.assertEquals(12345, decoded); - } finally { - Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); - } + public void testEncodeToDirectMemory() throws Exception { + assertMemoryLeak(() -> { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + long endAddr = QwpVarint.encode(addr, 12345); + int len = (int) (endAddr - addr); + Assert.assertTrue(len > 0); + + // Read back and verify + long decoded = QwpVarint.decode(addr, endAddr); + Assert.assertEquals(12345, decoded); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + }); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java index 4bf7eb5..dac1ec4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; @@ -42,614 +43,682 @@ public class WebSocketFrameParserTest { @Test - public void testControlFrameBetweenFragments() { - long buf = allocateBuffer(64); - try { - WebSocketFrameParser parser = new WebSocketFrameParser(); - - // First data fragment - writeBytes(buf, (byte) 0x01, (byte) 0x02, (byte) 'H', (byte) 'i'); - parser.parse(buf, buf + 4); - Assert.assertFalse(parser.isFin()); - Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); - - // Ping in the middle (control frame, FIN must be 1) - parser.reset(); - writeBytes(buf, (byte) 0x89, (byte) 0x00); - parser.parse(buf, buf + 2); - Assert.assertTrue(parser.isFin()); - Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); - - // Final data fragment - parser.reset(); - writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); - parser.parse(buf, buf + 3); - Assert.assertTrue(parser.isFin()); - Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); - } finally { - freeBuffer(buf, 64); - } - } + public void testControlFrameBetweenFragments() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(64); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); - @Test - public void testOpcodeIsControlFrame() { - Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.CONTINUATION)); - Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.TEXT)); - Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.BINARY)); - Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.CLOSE)); - Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PING)); - Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PONG)); + // First data fragment + writeBytes(buf, (byte) 0x01, (byte) 0x02, (byte) 'H', (byte) 'i'); + parser.parse(buf, buf + 4); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + + // Ping in the middle (control frame, FIN must be 1) + parser.reset(); + writeBytes(buf, (byte) 0x89, (byte) 0x00); + parser.parse(buf, buf + 2); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + + // Final data fragment + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + parser.parse(buf, buf + 3); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + }); } @Test - public void testOpcodeIsDataFrame() { - Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.CONTINUATION)); - Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.TEXT)); - Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.BINARY)); - Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.CLOSE)); - Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PING)); - Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PONG)); + public void testOpcodeIsControlFrame() throws Exception { + assertMemoryLeak(() -> { + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.TEXT)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.BINARY)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PONG)); + }); } @Test - public void testOpcodeIsValid() { - Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CONTINUATION)); - Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.TEXT)); - Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.BINARY)); - Assert.assertFalse(WebSocketOpcode.isValid(3)); - Assert.assertFalse(WebSocketOpcode.isValid(4)); - Assert.assertFalse(WebSocketOpcode.isValid(5)); - Assert.assertFalse(WebSocketOpcode.isValid(6)); - Assert.assertFalse(WebSocketOpcode.isValid(7)); - Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CLOSE)); - Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PING)); - Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PONG)); - Assert.assertFalse(WebSocketOpcode.isValid(0xB)); - Assert.assertFalse(WebSocketOpcode.isValid(0xF)); + public void testOpcodeIsDataFrame() throws Exception { + assertMemoryLeak(() -> { + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.CLOSE)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PING)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PONG)); + }); } @Test - public void testParse16BitLength() { - int payloadLen = 1000; - long buf = allocateBuffer(payloadLen + 16); - try { - writeBytes(buf, - (byte) 0x82, // FIN + BINARY - (byte) 126, // 16-bit length follows - (byte) (payloadLen >> 8), // Length high byte - (byte) (payloadLen & 0xFF) // Length low byte - ); - - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 4 + payloadLen); - - Assert.assertEquals(4 + payloadLen, consumed); - Assert.assertEquals(payloadLen, parser.getPayloadLength()); - } finally { - freeBuffer(buf, payloadLen + 16); - } + public void testOpcodeIsValid() throws Exception { + assertMemoryLeak(() -> { + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isValid(3)); + Assert.assertFalse(WebSocketOpcode.isValid(4)); + Assert.assertFalse(WebSocketOpcode.isValid(5)); + Assert.assertFalse(WebSocketOpcode.isValid(6)); + Assert.assertFalse(WebSocketOpcode.isValid(7)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PONG)); + Assert.assertFalse(WebSocketOpcode.isValid(0xB)); + Assert.assertFalse(WebSocketOpcode.isValid(0xF)); + }); } @Test - public void testParse64BitLength() { - long payloadLen = 70_000L; - long buf = allocateBuffer((int) payloadLen + 16); - try { - Unsafe.getUnsafe().putByte(buf, (byte) 0x82); - Unsafe.getUnsafe().putByte(buf + 1, (byte) 127); - Unsafe.getUnsafe().putLong(buf + 2, Long.reverseBytes(payloadLen)); + public void testParse16BitLength() throws Exception { + assertMemoryLeak(() -> { + int payloadLen = 1000; + long buf = allocateBuffer(payloadLen + 16); + try { + writeBytes(buf, + (byte) 0x82, // FIN + BINARY + (byte) 126, // 16-bit length follows + (byte) (payloadLen >> 8), // Length high byte + (byte) (payloadLen & 0xFF) // Length low byte + ); - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 10 + payloadLen); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4 + payloadLen); - Assert.assertEquals(10 + payloadLen, consumed); - Assert.assertEquals(payloadLen, parser.getPayloadLength()); - } finally { - freeBuffer(buf, (int) payloadLen + 16); - } + Assert.assertEquals(4 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); + } finally { + freeBuffer(buf, payloadLen + 16); + } + }); } @Test - public void testParse7BitLength() { - for (int len = 0; len <= 125; len++) { - long buf = allocateBuffer(256); + public void testParse64BitLength() throws Exception { + assertMemoryLeak(() -> { + long payloadLen = 70_000L; + long buf = allocateBuffer((int) payloadLen + 16); try { - writeBytes(buf, (byte) 0x82, (byte) len); - for (int i = 0; i < len; i++) { - Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); - } + Unsafe.getUnsafe().putByte(buf, (byte) 0x82); + Unsafe.getUnsafe().putByte(buf + 1, (byte) 127); + Unsafe.getUnsafe().putLong(buf + 2, Long.reverseBytes(payloadLen)); WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 2 + len); + int consumed = parser.parse(buf, buf + 10 + payloadLen); - Assert.assertEquals(2 + len, consumed); - Assert.assertEquals(len, parser.getPayloadLength()); + Assert.assertEquals(10 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); } finally { - freeBuffer(buf, 256); + freeBuffer(buf, (int) payloadLen + 16); } - } + }); } @Test - public void testParseBinaryFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 0x00); + public void testParse7BitLength() throws Exception { + assertMemoryLeak(() -> { + for (int len = 0; len <= 125; len++) { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x82, (byte) len); + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2 + len); - Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(2 + len, consumed); + Assert.assertEquals(len, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 256); + } + } + }); } @Test - public void testParseCloseFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, - (byte) 0x88, // FIN + CLOSE - (byte) 0x02, // Length 2 (just the code) - (byte) 0x03, (byte) 0xE8 // 1000 in big-endian - ); + public void testParseBinaryFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 4); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); - Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); - Assert.assertEquals(2, parser.getPayloadLength()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseCloseFrameEmpty() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x88, (byte) 0x00); + public void testParseCloseFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, + (byte) 0x88, // FIN + CLOSE + (byte) 0x02, // Length 2 (just the code) + (byte) 0x03, (byte) 0xE8 // 1000 in big-endian + ); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); - Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); - Assert.assertEquals(0, parser.getPayloadLength()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseCloseFrameWithReason() { - long buf = allocateBuffer(64); - try { - String reason = "Normal closure"; - byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + public void testParseCloseFrameEmpty() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x00); - Unsafe.getUnsafe().putByte(buf, (byte) 0x88); - Unsafe.getUnsafe().putByte(buf + 1, (byte) (2 + reasonBytes.length)); - Unsafe.getUnsafe().putShort(buf + 2, Short.reverseBytes((short) 1000)); - for (int i = 0; i < reasonBytes.length; i++) { - Unsafe.getUnsafe().putByte(buf + 4 + i, reasonBytes[i]); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); } + }); + } - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 4 + reasonBytes.length); + @Test + public void testParseCloseFrameWithReason() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(64); + try { + String reason = "Normal closure"; + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + + Unsafe.getUnsafe().putByte(buf, (byte) 0x88); + Unsafe.getUnsafe().putByte(buf + 1, (byte) (2 + reasonBytes.length)); + Unsafe.getUnsafe().putShort(buf + 2, Short.reverseBytes((short) 1000)); + for (int i = 0; i < reasonBytes.length; i++) { + Unsafe.getUnsafe().putByte(buf + 4 + i, reasonBytes[i]); + } - Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); - Assert.assertEquals(2 + reasonBytes.length, parser.getPayloadLength()); - } finally { - freeBuffer(buf, 64); - } + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4 + reasonBytes.length); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2 + reasonBytes.length, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 64); + } + }); } @Test - public void testParseContinuationFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x00, (byte) 0x05, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04, (byte) 0x05); + public void testParseContinuationFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x00, (byte) 0x05, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04, (byte) 0x05); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 7); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); - Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); - Assert.assertFalse(parser.isFin()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + Assert.assertFalse(parser.isFin()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseEmptyBuffer() { - long buf = allocateBuffer(16); - try { - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf); + public void testParseEmptyBuffer() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf); - Assert.assertEquals(0, consumed); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseEmptyPayload() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 0x00); + public void testParseEmptyPayload() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2); - Assert.assertEquals(2, consumed); - Assert.assertEquals(0, parser.getPayloadLength()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(2, consumed); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseFragmentedMessage() { - long buf = allocateBuffer(64); - try { - // First fragment: opcode=TEXT, FIN=0 - writeBytes(buf, (byte) 0x01, (byte) 0x03, (byte) 'H', (byte) 'e', (byte) 'l'); + public void testParseFragmentedMessage() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(64); + try { + // First fragment: opcode=TEXT, FIN=0 + writeBytes(buf, (byte) 0x01, (byte) 0x03, (byte) 'H', (byte) 'e', (byte) 'l'); - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 5); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 5); - Assert.assertEquals(5, consumed); - Assert.assertFalse(parser.isFin()); - Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + Assert.assertEquals(5, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); - // Continuation: opcode=CONTINUATION, FIN=0 - parser.reset(); - writeBytes(buf, (byte) 0x00, (byte) 0x02, (byte) 'l', (byte) 'o'); - consumed = parser.parse(buf, buf + 4); + // Continuation: opcode=CONTINUATION, FIN=0 + parser.reset(); + writeBytes(buf, (byte) 0x00, (byte) 0x02, (byte) 'l', (byte) 'o'); + consumed = parser.parse(buf, buf + 4); - Assert.assertEquals(4, consumed); - Assert.assertFalse(parser.isFin()); - Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + Assert.assertEquals(4, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); - // Final fragment: opcode=CONTINUATION, FIN=1 - parser.reset(); - writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); - consumed = parser.parse(buf, buf + 3); + // Final fragment: opcode=CONTINUATION, FIN=1 + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + consumed = parser.parse(buf, buf + 3); - Assert.assertEquals(3, consumed); - Assert.assertTrue(parser.isFin()); - Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); - } finally { - freeBuffer(buf, 64); - } + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + }); } @Test - public void testParseIncompleteHeader16BitLength() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 126, (byte) 0x01); + public void testParseIncompleteHeader16BitLength() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 126, (byte) 0x01); - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 3); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); - Assert.assertEquals(0, consumed); - Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseIncompleteHeader1Byte() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82); + public void testParseIncompleteHeader1Byte() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82); - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 1); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 1); - Assert.assertEquals(0, consumed); - Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseIncompleteHeader64BitLength() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 127, (byte) 0, (byte) 0, (byte) 0, (byte) 0); + public void testParseIncompleteHeader64BitLength() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 127, (byte) 0, (byte) 0, (byte) 0, (byte) 0); - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 6); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 6); - Assert.assertEquals(0, consumed); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseIncompletePayload() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 0x05, (byte) 0x01, (byte) 0x02); - - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 4); + public void testParseIncompletePayload() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x05, (byte) 0x01, (byte) 0x02); - Assert.assertEquals(2, consumed); - Assert.assertEquals(5, parser.getPayloadLength()); - Assert.assertEquals(WebSocketFrameParser.STATE_NEED_PAYLOAD, parser.getState()); - } finally { - freeBuffer(buf, 16); - } - } + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4); - @Test - public void testParseMaxControlFrameSize() { - long buf = allocateBuffer(256); - try { - Unsafe.getUnsafe().putByte(buf, (byte) 0x89); // PING - Unsafe.getUnsafe().putByte(buf + 1, (byte) 125); - for (int i = 0; i < 125; i++) { - Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + Assert.assertEquals(2, consumed); + Assert.assertEquals(5, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_PAYLOAD, parser.getState()); + } finally { + freeBuffer(buf, 16); } - - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 127); - - Assert.assertEquals(127, consumed); - Assert.assertEquals(125, parser.getPayloadLength()); - Assert.assertNotEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - } finally { - freeBuffer(buf, 256); - } + }); } @Test - public void testParseMinimalFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 0x01, (byte) 0xFF); + public void testParseMaxControlFrameSize() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(256); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x89); // PING + Unsafe.getUnsafe().putByte(buf + 1, (byte) 125); + for (int i = 0; i < 125; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } - WebSocketFrameParser parser = new WebSocketFrameParser(); - int consumed = parser.parse(buf, buf + 3); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 127); - Assert.assertEquals(3, consumed); - Assert.assertTrue(parser.isFin()); - Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); - Assert.assertEquals(1, parser.getPayloadLength()); - Assert.assertFalse(parser.isMasked()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(127, consumed); + Assert.assertEquals(125, parser.getPayloadLength()); + Assert.assertNotEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 256); + } + }); } @Test - public void testParseMultipleFramesInBuffer() { - long buf = allocateBuffer(32); - try { - writeBytes(buf, - (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02, - (byte) 0x81, (byte) 0x03, (byte) 'a', (byte) 'b', (byte) 'c' - ); - - WebSocketFrameParser parser = new WebSocketFrameParser(); + public void testParseMinimalFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x01, (byte) 0xFF); - int consumed = parser.parse(buf, buf + 9); - Assert.assertEquals(4, consumed); - Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); - Assert.assertEquals(2, parser.getPayloadLength()); + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); - parser.reset(); - consumed = parser.parse(buf + 4, buf + 9); - Assert.assertEquals(5, consumed); - Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); - Assert.assertEquals(3, parser.getPayloadLength()); - } finally { - freeBuffer(buf, 32); - } + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(1, parser.getPayloadLength()); + Assert.assertFalse(parser.isMasked()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParsePingFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x89, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + public void testParseMultipleFramesInBuffer() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(32); + try { + writeBytes(buf, + (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02, + (byte) 0x81, (byte) 0x03, (byte) 'a', (byte) 'b', (byte) 'c' + ); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 6); + WebSocketFrameParser parser = new WebSocketFrameParser(); - Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); - Assert.assertEquals(4, parser.getPayloadLength()); - } finally { - freeBuffer(buf, 16); - } + int consumed = parser.parse(buf, buf + 9); + Assert.assertEquals(4, consumed); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + consumed = parser.parse(buf + 4, buf + 9); + Assert.assertEquals(5, consumed); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + Assert.assertEquals(3, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 32); + } + }); } @Test - public void testParsePongFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x8A, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + public void testParsePingFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x89, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 6); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); - Assert.assertEquals(WebSocketOpcode.PONG, parser.getOpcode()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + Assert.assertEquals(4, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testParseTextFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x81, (byte) 0x00); + public void testParsePongFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x8A, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); - Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketOpcode.PONG, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testRejectCloseFrameWith1BytePayload() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x88, (byte) 0x01, (byte) 0x00); + public void testParseTextFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x81, (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 3); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); - Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testRejectFragmentedControlFrame() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x09, (byte) 0x00); + public void testRejectCloseFrameWith1BytePayload() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x01, (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 3); - Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testRejectMaskedFrame() { - // Client-side parser rejects masked frames from the server - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 0x81, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0xFF); + public void testRejectFragmentedControlFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x09, (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 7); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); - Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testRejectOversizeControlFrame() { - long buf = allocateBuffer(256); - try { - writeBytes(buf, (byte) 0x89, (byte) 126, (byte) 0x00, (byte) 0x7E); + public void testRejectMaskedFrame() throws Exception { + assertMemoryLeak(() -> { + // Client-side parser rejects masked frames from the server + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x81, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0xFF); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 4); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); - Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); - } finally { - freeBuffer(buf, 256); - } + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testRejectRSV2Bit() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0xA2, (byte) 0x00); + public void testRejectOversizeControlFrame() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x89, (byte) 126, (byte) 0x00, (byte) 0x7E); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); - Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 256); + } + }); } @Test - public void testRejectRSV3Bit() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x92, (byte) 0x00); + public void testRejectRSV2Bit() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0xA2, (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); - Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testRejectReservedBits() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0xC2, (byte) 0x00); + public void testRejectRSV3Bit() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x92, (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 2); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); - Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); - Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); - } finally { - freeBuffer(buf, 16); - } + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); } @Test - public void testRejectUnknownOpcode() { - for (int opcode : new int[]{3, 4, 5, 6, 7, 0xB, 0xC, 0xD, 0xE, 0xF}) { + public void testRejectReservedBits() throws Exception { + assertMemoryLeak(() -> { long buf = allocateBuffer(16); try { - writeBytes(buf, (byte) (0x80 | opcode), (byte) 0x00); + writeBytes(buf, (byte) 0xC2, (byte) 0x00); WebSocketFrameParser parser = new WebSocketFrameParser(); parser.parse(buf, buf + 2); - Assert.assertEquals("Opcode " + opcode + " should be rejected", - WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); } finally { freeBuffer(buf, 16); } - } + }); } @Test - public void testReset() { - long buf = allocateBuffer(16); - try { - writeBytes(buf, (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02); + public void testRejectUnknownOpcode() throws Exception { + assertMemoryLeak(() -> { + for (int opcode : new int[]{3, 4, 5, 6, 7, 0xB, 0xC, 0xD, 0xE, 0xF}) { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) (0x80 | opcode), (byte) 0x00); - WebSocketFrameParser parser = new WebSocketFrameParser(); - parser.parse(buf, buf + 4); + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); - Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); - Assert.assertEquals(2, parser.getPayloadLength()); + Assert.assertEquals("Opcode " + opcode + " should be rejected", + WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + }); + } - parser.reset(); + @Test + public void testReset() throws Exception { + assertMemoryLeak(() -> { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02); - Assert.assertEquals(0, parser.getOpcode()); - Assert.assertEquals(0, parser.getPayloadLength()); - Assert.assertEquals(WebSocketFrameParser.STATE_HEADER, parser.getState()); - } finally { - freeBuffer(buf, 16); - } + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + + Assert.assertEquals(0, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_HEADER, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + }); } private static long allocateBuffer(int size) { diff --git a/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java b/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java index c056324..b45cbc6 100644 --- a/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java +++ b/core/src/test/java/io/questdb/client/test/std/VectFuzzTest.java @@ -29,14 +29,14 @@ import io.questdb.client.std.Os; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; -import io.questdb.client.test.tools.TestUtils; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; public class VectFuzzTest { @Test public void testMemmove() throws Exception { - TestUtils.assertMemoryLeak(() -> { + assertMemoryLeak(() -> { int maxSize = 1024 * 1024; int[] sizes = {1024, 4096, maxSize}; int buffSize = 1024 + 4096 + maxSize; From 874d3cf7cfbccf887152a8fe50a9bb2949e559d9 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 11:10:11 +0100 Subject: [PATCH 093/230] Add column type mismatch tests Add unit tests for QwpTableBuffer.getOrCreateColumn() when called with a conflicting type for an existing column. Two tests cover both code paths: the fast path (sequential cursor hit) and the slow path (hash-map fallback). Both verify that LineSenderException is thrown with the correct message. Co-Authored-By: Claude Opus 4.6 --- .../qwp/protocol/QwpTableBufferTest.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 47806b0..a1a7d0d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -405,6 +405,54 @@ public void testDoubleArrayWrapperVaryingDimensionality() throws Exception { }); } + @Test + public void testGetOrCreateColumnConflictingTypeFastPath() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // First call creates the column as LONG + table.getOrCreateColumn("x", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + + // Second call with the same name but a different type hits the fast path + // (sequential cursor matches the column name) and must throw + try { + table.getOrCreateColumn("x", QwpConstants.TYPE_DOUBLE, false); + fail("Expected LineSenderException for column type mismatch"); + } catch (LineSenderException e) { + assertEquals( + "Column type mismatch for x: existing=" + QwpConstants.TYPE_LONG + " new=" + QwpConstants.TYPE_DOUBLE, + e.getMessage() + ); + } + } + }); + } + + @Test + public void testGetOrCreateColumnConflictingTypeSlowPath() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Create two columns so the fast-path cursor can be defeated + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1L); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v"); + table.nextRow(); + + // Access column "b" first — cursor now expects "a" at index 0, + // but we ask for "b", so the fast path misses and falls through + // to the hash-map lookup, which must detect the type conflict + try { + table.getOrCreateColumn("b", QwpConstants.TYPE_LONG, false); + fail("Expected LineSenderException for column type mismatch"); + } catch (LineSenderException e) { + assertEquals( + "Column type mismatch for b: existing=" + QwpConstants.TYPE_STRING + " new=" + QwpConstants.TYPE_LONG, + e.getMessage() + ); + } + } + }); + } + @Test public void testLongArrayMultipleRows() throws Exception { assertMemoryLeak(() -> { From 3e6bdd4e56c1c1b226e042ca78b17e72f688a6b5 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 11:19:11 +0100 Subject: [PATCH 094/230] Reject HTTP-specific builder options for WebSocket The LineSenderBuilder silently accepted HTTP-specific options (httpPath, httpTimeout, retryTimeout, minRequestThroughput, maxBackoff, protocolVersion, disableAutoFlush) when the transport was WEBSOCKET. validateParameters() now rejects these with clear error messages, matching the pattern already used for TCP. configureDefaults() now sets the maxBackoffMillis default only for HTTP, so validateParameters() can detect user-explicit values. Seven @Ignore'd tests in LineSenderBuilderWebSocketTest are converted to active tests that verify the builder throws. Co-Authored-By: Claude Opus 4.6 --- .../main/java/io/questdb/client/Sender.java | 23 ++++- .../LineSenderBuilderWebSocketTest.java | 84 +++++++++---------- 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 33d2df2..c815bf7 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1414,7 +1414,7 @@ private void configureDefaults() { if (maxNameLength == PARAMETER_NOT_SET_EXPLICITLY) { maxNameLength = DEFAULT_MAX_NAME_LEN; } - if (maxBackoffMillis == PARAMETER_NOT_SET_EXPLICITLY) { + if (maxBackoffMillis == PARAMETER_NOT_SET_EXPLICITLY && protocol == PROTOCOL_HTTP) { maxBackoffMillis = DEFAULT_MAX_BACKOFF_MILLIS; } } @@ -1765,6 +1765,27 @@ private void validateParameters() { if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { throw new LineSenderException("in-flight window size requires async mode"); } + if (httpPath != null) { + throw new LineSenderException("HTTP path is not supported for WebSocket protocol"); + } + if (httpTimeout != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("HTTP timeout is not supported for WebSocket protocol"); + } + if (retryTimeoutMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("retry timeout is not supported for WebSocket protocol"); + } + if (minRequestThroughput != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("minimum request throughput is not supported for WebSocket protocol"); + } + if (maxBackoffMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("max backoff is not supported for WebSocket protocol"); + } + if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol version is not supported for WebSocket protocol"); + } + if (autoFlushIntervalMillis == Integer.MAX_VALUE) { + throw new LineSenderException("disabling auto-flush is not supported for WebSocket protocol"); + } } else { throw new LineSenderException("unsupported protocol ") .put("[protocol=").put(protocol).put("]"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 378158b..c1b58f7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -279,12 +279,12 @@ public void testCustomTrustStore_butTlsNotEnabled_fails() { } @Test - @Ignore("Disable auto flush may need different semantics for WebSocket") - public void testDisableAutoFlush_semantics() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .disableAutoFlush(); - Assert.assertNotNull(builder); + public void testDisableAutoFlush_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush(), + "not supported for WebSocket"); } @Test @@ -343,21 +343,21 @@ public void testFullAsyncConfigurationWithTls() { } @Test - @Ignore("HTTP path is HTTP-specific and may not apply to WebSocket") - public void testHttpPath_mayNotApply() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .httpPath("/custom/path"); - Assert.assertNotNull(builder); + public void testHttpPath_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpPath("/custom/path"), + "not supported for WebSocket"); } @Test - @Ignore("HTTP timeout is HTTP-specific and may not apply to WebSocket") - public void testHttpTimeout_mayNotApply() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .httpTimeoutMillis(5000); - Assert.assertNotNull(builder); + public void testHttpTimeout_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpTimeoutMillis(5000), + "not supported for WebSocket"); } @Test @@ -443,12 +443,12 @@ public void testMalformedPortInAddress_fails() { } @Test - @Ignore("Max backoff is HTTP-specific and may not apply to WebSocket") - public void testMaxBackoff_mayNotApply() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .maxBackoffMillis(1000); - Assert.assertNotNull(builder); + public void testMaxBackoff_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxBackoffMillis(1000), + "not supported for WebSocket"); } @Test @@ -477,12 +477,12 @@ public void testMaxNameLengthTooSmall_fails() { } @Test - @Ignore("Min request throughput is HTTP-specific and may not apply to WebSocket") - public void testMinRequestThroughput_mayNotApply() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .minRequestThroughput(10000); - Assert.assertNotNull(builder); + public void testMinRequestThroughput_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .minRequestThroughput(10000), + "not supported for WebSocket"); } @Test @@ -511,21 +511,21 @@ public void testPortMismatch_fails() { } @Test - @Ignore("Protocol version is for ILP text protocol, WebSocket uses ILP v4 binary protocol") - public void testProtocolVersion_notApplicable() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .protocolVersion(Sender.PROTOCOL_VERSION_V2); - Assert.assertNotNull(builder); + public void testProtocolVersion_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .protocolVersion(Sender.PROTOCOL_VERSION_V2), + "not supported for WebSocket"); } @Test - @Ignore("Retry timeout is HTTP-specific and may not apply to WebSocket") - public void testRetryTimeout_mayNotApply() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .retryTimeoutMillis(5000); - Assert.assertNotNull(builder); + public void testRetryTimeout_notSupportedForWebSocket() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .retryTimeoutMillis(5000), + "not supported for WebSocket"); } @Test From 6d30696ec7c12a37686b87766ed927d2669f7df2 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 12:10:08 +0100 Subject: [PATCH 095/230] Replace Thread.sleep with thread state checks Replace Thread.sleep(100) calls in InFlightWindowTest and WebSocketSendQueueTest with an awaitThreadBlocked() helper that spins on Thread.getState(). The helper returns immediately once WAITING/TIMED_WAITING is detected, instead of relying on a fixed 100ms sleep that is unreliable on loaded CI machines. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/InFlightWindowTest.java | 24 ++++++++++++++----- .../qwp/client/WebSocketSendQueueTest.java | 14 ++++++++++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java index f31956b..b3fb639 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -146,7 +146,7 @@ public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { waitThread.start(); assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); + awaitThreadBlocked(waitThread); assertTrue(waiting.get()); // Single cumulative ACK clears all @@ -181,7 +181,7 @@ public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { addThread.start(); assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Give time to block + awaitThreadBlocked(addThread); assertTrue(blocked.get()); // Cumulative ACK frees multiple slots @@ -215,7 +215,7 @@ public void testAwaitEmpty() throws Exception { waitThread.start(); assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); + awaitThreadBlocked(waitThread); assertTrue(waiting.get()); // Cumulative ACK all batches @@ -494,7 +494,7 @@ public void testFailWakesAwaitEmpty() throws Exception { waitThread.start(); assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Let it block + awaitThreadBlocked(waitThread); // Fail a batch - should wake the blocked thread window.fail(0, new RuntimeException("Test error")); @@ -528,7 +528,7 @@ public void testFailWakesBlockedAdder() throws Exception { addThread.start(); assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Let it block + awaitThreadBlocked(addThread); // Fail a batch - should wake the blocked thread window.fail(0, new RuntimeException("Test error")); @@ -787,7 +787,7 @@ public void testWindowBlocksWhenFull() throws Exception { // Wait for thread to start and block assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Give time to block + awaitThreadBlocked(addThread); assertTrue(blocked.get()); // Free a slot @@ -827,4 +827,16 @@ public void testZeroBatchId() { assertTrue(window.acknowledge(0)); assertTrue(window.isEmpty()); } + + private static void awaitThreadBlocked(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + Thread.State state = thread.getState(); + if (state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING) { + return; + } + Thread.sleep(1); + } + fail("Thread did not reach blocked state within 5s, state: " + thread.getState()); + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java index 022bfde..db97192 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java @@ -109,7 +109,7 @@ public void testEnqueueWaitsUntilSlotAvailable() throws Exception { t.start(); assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); + awaitThreadBlocked(t); assertEquals("Second enqueue should still be waiting", 1, finished.getCount()); // Free space so I/O thread can poll pending slot. @@ -266,6 +266,18 @@ public void testFlushFailsWhenServerClosesConnection() throws Exception { }); } + private static void awaitThreadBlocked(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + Thread.State state = thread.getState(); + if (state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING) { + return; + } + Thread.sleep(1); + } + fail("Thread did not reach blocked state within 5s, state: " + thread.getState()); + } + private static void closeQuietly(WebSocketSendQueue queue) { if (queue != null) { queue.close(); From 33e5796304f4612b02262b6ee52b5144ff10ce61 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 12:17:43 +0100 Subject: [PATCH 096/230] Add sleep to stress test acker spin loop The acker thread in testHighConcurrencyStress() spun in a tight loop with no sleep when there was nothing new to acknowledge, wasting CPU. Add Thread.sleep(1) in the else branch, matching the pattern already used in testConcurrentAddAndCumulativeAck(). Co-Authored-By: Claude Opus 4.6 --- .../client/test/cutlass/qwp/client/InFlightWindowTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java index b3fb639..36d098d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -612,8 +612,9 @@ public void testHighConcurrencyStress() throws Exception { if (highest > lastAcked) { window.acknowledgeUpTo(highest); lastAcked = highest; + } else { + Thread.sleep(1); } - // No sleep - maximum contention } } catch (Throwable t) { error.set(t); From 932cafe3a6782b6b7935b6dbc40d0b14a906e86c Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 12:28:03 +0100 Subject: [PATCH 097/230] Document TOCTOU race in findUnusedPort() Add a comment to findUnusedPort() explaining why the TOCTOU race between closing the ServerSocket and the caller's connect attempt is acceptable. Every caller is a negative test that expects the connection to fail, so a stolen port produces the same outcome. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/LineSenderBuilderWebSocketTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index c1b58f7..ae6480c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -725,6 +725,11 @@ private static void assertThrowsAny(Runnable action, String... anyOf) { } } + // There is a TOCTOU race between closing the ServerSocket and the caller's + // connect attempt — another process could bind the port in between. This is + // acceptable because every caller is a negative test that expects the connection + // to fail. If the port is stolen, the test connects to a non-QuestDB endpoint, + // which also fails with the same error. private static int findUnusedPort() throws Exception { try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { return s.getLocalPort(); From 8e721effc046494230e9956b33e5c3d695d83e29 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 12:34:16 +0100 Subject: [PATCH 098/230] Bump javadoc plugin source level to Java 17 The maven-javadoc-plugin had 11 in both the "javadoc" and "maven-central-release" profiles, while the compiler targets Java 17. This mismatch caused javadoc generation to fail on Java 17 syntax such as switch expressions with arrow labels. Co-Authored-By: Claude Opus 4.6 --- core/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index a6dfa9f..5f052c6 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -198,7 +198,7 @@ none - 11 + 17 false ${compilerArg1} @@ -296,7 +296,7 @@ none - 11 + 17 false ${compilerArg1} From 2703bf47e931c9ba2fcdeb3f1f38aa9ace918ab9 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 12:46:43 +0100 Subject: [PATCH 099/230] Upgrade Java 11 references to Java 17 The Javadoc CI step installed Java 11, but the javadoc plugin now targets source level 17, causing "invalid source release: 17" failures. Update the Javadoc validation step in run_tests_pipeline.yaml to install Java 17. Also update the Maven profile in core/pom.xml: rename the profile from java11+ to java17+, bump jdk.version and java.enforce.version to 17, and change the activation range from (11,) to [17,). Co-Authored-By: Claude Opus 4.6 --- ci/run_tests_pipeline.yaml | 4 ++-- core/pom.xml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 69c3653..8bc180b 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -30,9 +30,9 @@ stages: displayName: "Checkout PR source branch" condition: eq(variables['Build.Reason'], 'PullRequest') - task: JavaToolInstaller@0 - displayName: "Install Java 11" + displayName: "Install Java 17" inputs: - versionSpec: "11" + versionSpec: "17" jdkArchitectureOption: "x64" jdkSourceOption: "PreInstalled" - bash: mvn -f core/pom.xml javadoc:javadoc -Pjavadoc --batch-mode diff --git a/core/pom.xml b/core/pom.xml index 5f052c6..a117725 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -384,20 +384,20 @@ - java11+ + java17+ - 11 - 11 + 17 + 17 questdb --add-exports java.base/jdk.internal.math=io.questdb.client - nothing-to-exclude-dummy-value-include-all-java11plus - nothing-to-exclude-dummy-value-include-all-java11plus + nothing-to-exclude-dummy-value-include-all-java17plus + nothing-to-exclude-dummy-value-include-all-java17plus ${javac.target} ${javac.target} - (11,) + [17,) From ecc73b6ed3c5ba48556cc924a1138da2e543839a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 13:08:16 +0100 Subject: [PATCH 100/230] Switch CI to clone jh_experiment_new_ilp branch The CI pipeline cloned the rd_strip_ilp_sender branch of QuestDB to build the server for integration tests. Update the branch reference to jh_experiment_new_ilp, which contains the current QWP/WebSocket server support needed by QwpSenderTest. Co-Authored-By: Claude Opus 4.6 --- ci/run_tests_pipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 8bc180b..c4e74c9 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -72,8 +72,8 @@ stages: lfs: false submodules: false - template: setup.yaml - # TODO: remove branch once rd_strip_ilp_sender is merged - - script: git clone --depth 1 -b rd_strip_ilp_sender https://github.com/questdb/questdb.git ./questdb + # TODO: remove branch once jh_experiment_new_ilp is merged + - script: git clone --depth 1 -b jh_experiment_new_ilp https://github.com/questdb/questdb.git ./questdb displayName: git clone questdb - task: Maven@3 displayName: "Update client version" From 19eabb05f15f4a01e217e31f6dfd6ca5f2fd6d36 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 13:19:51 +0100 Subject: [PATCH 101/230] Fix client artifact ID in CI version overrides The client module's artifactId was renamed from "client" to "questdb-client", but the CI pipeline's versions:use-dep-version steps still filtered on org.questdb:client. The filter never matched, so QuestDB's build tried to fetch the unreleased questdb-client:1.0.2-SNAPSHOT from Maven Central instead of using the locally installed 9.9.9 jar. Co-Authored-By: Claude Opus 4.6 --- ci/run_tests_pipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index c4e74c9..9fbede2 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -95,21 +95,21 @@ stages: mavenPOMFile: "questdb/core/pom.xml" jdkVersionOption: "default" goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" + options: "-Dincludes=org.questdb:questdb-client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" - task: Maven@3 displayName: "Update QuestDB client version: benchmarks" inputs: mavenPOMFile: "questdb/benchmarks/pom.xml" jdkVersionOption: "default" goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" + options: "-Dincludes=org.questdb:questdb-client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" - task: Maven@3 displayName: "Update QuestDB client version: utils" inputs: mavenPOMFile: "questdb/utils/pom.xml" jdkVersionOption: "default" goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" + options: "-Dincludes=org.questdb:questdb-client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" - task: Maven@3 displayName: "Compile QuestDB" inputs: From afc654fa51ee83f9078944dc225004ff9fcd92e2 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 13:57:12 +0100 Subject: [PATCH 102/230] Fix CI client version override for property refs The versions:use-dep-version Maven plugin silently skips dependencies whose version is a property reference (confirmed by its own log: "Ignoring a dependency with the version set using a property"). Since QuestDB's POMs define the client version via ${questdb.client.version}, the three Maven override steps never took effect, leaving 1.0.2-SNAPSHOT unresolved. Replace the three versions:use-dep-version tasks with a single sed command that directly updates the property in core, benchmarks, and utils POMs. Co-Authored-By: Claude Opus 4.6 --- ci/run_tests_pipeline.yaml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 9fbede2..d36ed0d 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -89,27 +89,10 @@ stages: jdkVersionOption: "default" goals: "install" options: "-DskipTests --batch-mode" - - task: Maven@3 - displayName: "Update QuestDB client version: core" - inputs: - mavenPOMFile: "questdb/core/pom.xml" - jdkVersionOption: "default" - goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:questdb-client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" - - task: Maven@3 - displayName: "Update QuestDB client version: benchmarks" - inputs: - mavenPOMFile: "questdb/benchmarks/pom.xml" - jdkVersionOption: "default" - goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:questdb-client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" - - task: Maven@3 - displayName: "Update QuestDB client version: utils" - inputs: - mavenPOMFile: "questdb/utils/pom.xml" - jdkVersionOption: "default" - goals: "versions:use-dep-version" - options: "-Dincludes=org.questdb:questdb-client -DdepVersion=9.9.9 -DforceVersion=true --batch-mode" + - bash: | + sed -i 's|.*|9.9.9|' \ + questdb/core/pom.xml questdb/benchmarks/pom.xml questdb/utils/pom.xml + displayName: "Update QuestDB client version to 9.9.9" - task: Maven@3 displayName: "Compile QuestDB" inputs: From 90d93615b7f8f0b9a0e412542ff0871926700077 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 15:00:54 +0100 Subject: [PATCH 103/230] Enable HTTP server in CI configs The QWP WebSocket sender connects to QuestDB's HTTP port (9000) for the WebSocket upgrade at /write/v4. Both CI server configs had http.enabled=false, which prevented the HTTP listener from starting. PG wire tests (port 8812) were unaffected, but all QwpSenderTest cases failed with "Failed to connect to 127.0.0.1:9000". Set http.enabled=true in both default and authenticated configs. Co-Authored-By: Claude Opus 4.6 --- ci/confs/authenticated/server.conf | 2 +- ci/confs/default/server.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/confs/authenticated/server.conf b/ci/confs/authenticated/server.conf index d30f878..a1a19b1 100644 --- a/ci/confs/authenticated/server.conf +++ b/ci/confs/authenticated/server.conf @@ -40,7 +40,7 @@ config.validation.strict=true ################ HTTP settings ################## # enable HTTP server -http.enabled=false +http.enabled=true # IP address and port of HTTP server #http.net.bind.to=0.0.0.0:9000 diff --git a/ci/confs/default/server.conf b/ci/confs/default/server.conf index f96a35e..dfe5419 100644 --- a/ci/confs/default/server.conf +++ b/ci/confs/default/server.conf @@ -40,7 +40,7 @@ config.validation.strict=true ################ HTTP settings ################## # enable HTTP server -http.enabled=false +http.enabled=true # IP address and port of HTTP server #http.net.bind.to=0.0.0.0:9000 From 596d65ca8f38e06fdd5600362b4fb5745d02eeb3 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 15:20:09 +0100 Subject: [PATCH 104/230] Use packed format for nullable GeoHash columns QwpTableBuffer.addNull() had a special case for GEOHASH that wrote a placeholder 0L value and incremented valueCount even for null rows. This produced a dense wire format where all rows occupied space in the values array, inconsistent with every other nullable column type which uses packed format (non-null values only). Remove the GEOHASH special case so addNull() for GEOHASH columns follows the same packed convention as fixed-width, boolean, and decimal columns. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/protocol/QwpTableBuffer.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 7e21881..a61a3f0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -787,13 +787,6 @@ public void addNull() { if (nullable) { ensureNullCapacity(size + 1); markNull(size); - // GEOHASH uses dense wire format: all rows (including nulls) - // occupy space in the values array. Write a placeholder value - // so the data buffer stays aligned with the row index. - if (type == TYPE_GEOHASH) { - dataBuffer.putLong(0L); - valueCount++; - } size++; } else { // For non-nullable columns, store a sentinel/default value From cef8bd3aaaaeca82ead816c0babba67c0c0f55eb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 15:25:49 +0100 Subject: [PATCH 105/230] Fix sed portability for macOS CI runners BSD sed on macOS requires a backup extension argument after the -i flag. The bare `sed -i` syntax only works with GNU sed on Linux, causing the CI build to fail on macOS runners with "extra characters at the end of q command". Switch to `sed -i.bak` which both GNU and BSD sed accept, and clean up the resulting .bak files afterward. Co-Authored-By: Claude Opus 4.6 --- ci/run_tests_pipeline.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index d36ed0d..a430af4 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -90,8 +90,9 @@ stages: goals: "install" options: "-DskipTests --batch-mode" - bash: | - sed -i 's|.*|9.9.9|' \ + sed -i.bak 's|.*|9.9.9|' \ questdb/core/pom.xml questdb/benchmarks/pom.xml questdb/utils/pom.xml + rm -f questdb/core/pom.xml.bak questdb/benchmarks/pom.xml.bak questdb/utils/pom.xml.bak displayName: "Update QuestDB client version to 9.9.9" - task: Maven@3 displayName: "Compile QuestDB" From 7a39452bfb1efa484d0f3c837e6535a26469702d Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 27 Feb 2026 16:55:39 +0100 Subject: [PATCH 106/230] Rename CI job to clarify it runs server tests The "Run OSS line tests" display name was confusing because the client module is also OSS. Rename to "Run server line tests" to make it clear the job runs server-side line protocol tests from the parent questdb repo. Co-Authored-By: Claude Opus 4.6 --- ci/run_oss_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_oss_tests.yaml b/ci/run_oss_tests.yaml index 91cde3a..7e8feed 100644 --- a/ci/run_oss_tests.yaml +++ b/ci/run_oss_tests.yaml @@ -1,7 +1,7 @@ # Run the tests from OSS using the current client version steps: - task: Maven@3 - displayName: "Run OSS line tests" + displayName: "Run server line tests" inputs: mavenPOMFile: "questdb/pom.xml" jdkVersionOption: "default" From ea391bd7f06448bd1f3461e05ff0d2b12fa82d3f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 15:07:28 +0100 Subject: [PATCH 107/230] Remove dead recursive Net.send(long, long, int) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The send(long fd, long ptr, int len) overload called itself recursively instead of delegating to the native send(int, long, int), causing StackOverflowError on any invocation. No callers exist — all call sites pass an int fd, which Java resolves directly to the native method — so the overload is dead code. Remove it entirely. Co-Authored-By: Claude Opus 4.6 --- core/src/main/java/io/questdb/client/network/Net.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index a3f7939..fe21505 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -118,10 +118,6 @@ public static void init() { public static native int recv(int fd, long ptr, int len); - public static int send(long fd, long ptr, int len) { - return send(fd, ptr, len); - } - public static native int send(int fd, long ptr, int len); public native static int sendTo(int fd, long ptr, int len, long sockaddr); From 207222251beabaa4ac7d35d0319d897ee23b755a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 15:17:32 +0100 Subject: [PATCH 108/230] Fix native memory leak in WebSocketClient constructor The constructor allocates native memory in three steps: sendBuffer, controlFrameBuffer, and recvBufPtr. If a later allocation fails (e.g., OutOfMemoryError from Unsafe.malloc), previously allocated resources leak because the constructor never completes and close() is never called. Wrap the allocation sequence in a try/catch that uses Misc.free() to release any already-allocated buffers and the socket on failure before re-throwing the exception. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 78a3245..c4cc915 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -103,15 +103,22 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock int sendBufSize = Math.max(configuration.getInitialRequestBufferSize(), DEFAULT_SEND_BUFFER_SIZE); int maxSendBufSize = Math.max(configuration.getMaximumRequestBufferSize(), sendBufSize); - this.sendBuffer = new WebSocketSendBuffer(sendBufSize, maxSendBufSize); - // Control frames (ping/pong/close) have max 125-byte payload + 14-byte header. - // This dedicated buffer prevents sendPongFrame from clobbering an in-progress - // frame being built in the main sendBuffer. - this.controlFrameBuffer = new WebSocketSendBuffer(256, 256); - - this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE); - this.maxRecvBufSize = Math.max(configuration.getMaximumResponseBufferSize(), recvBufSize); - this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT); + try { + this.sendBuffer = new WebSocketSendBuffer(sendBufSize, maxSendBufSize); + // Control frames (ping/pong/close) have max 125-byte payload + 14-byte header. + // This dedicated buffer prevents sendPongFrame from clobbering an in-progress + // frame being built in the main sendBuffer. + this.controlFrameBuffer = new WebSocketSendBuffer(256, 256); + + this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE); + this.maxRecvBufSize = Math.max(configuration.getMaximumResponseBufferSize(), recvBufSize); + this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT); + } catch (Throwable t) { + Misc.free(controlFrameBuffer); + Misc.free(sendBuffer); + Misc.free(socket); + throw t; + } this.recvPos = 0; this.recvReadPos = 0; From dd2cc0bd29c8d0060b181fa3c1ddfc3d7b1d8c9f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 15:43:45 +0100 Subject: [PATCH 109/230] Fix sendQueue leak on close when flush fails Move sendQueue.close() out of the try block in QwpWebSocketSender.close() so the I/O thread always shuts down, even if flushPendingRows() or sealAndSwapBuffer() throws. Previously, an exception during flush caused sendQueue.close() to be skipped, leaving the I/O thread running while client.close() closed the socket from under it. The send queue now closes in its own try-catch before buffers and client, ensuring orderly shutdown. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/QwpWebSocketSender.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1cc8e51..c5442cd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -479,9 +479,6 @@ public void close() { if (inFlightWindow != null) { inFlightWindow.awaitEmpty(); } - if (sendQueue != null) { - sendQueue.close(); - } } else { // Sync mode (window=1): flush pending rows synchronously if (pendingRowCount > 0 && client != null && client.isConnected()) { @@ -492,6 +489,16 @@ public void close() { LOG.error("Error during close: {}", String.valueOf(e)); } + // Shut down the I/O thread before closing the socket or buffers + // it may be using. This must run even if the flush above failed. + if (sendQueue != null) { + try { + sendQueue.close(); + } catch (Exception e) { + LOG.error("Error closing send queue: {}", String.valueOf(e)); + } + } + // Close buffers (async mode only, window > 1) if (buffer0 != null) { buffer0.close(); From 937295c562282d5e90de85b5dcb0b4d8af8aef4b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 15:56:56 +0100 Subject: [PATCH 110/230] Fix final field init in WebSocketClient constructor The catch block in the constructor referenced the final fields controlFrameBuffer and sendBuffer, which may not have been assigned yet when an exception occurs. Use local variables initialized to null for the try block, then copy them to the final fields after the try-catch succeeds. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 37 ++++--------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index c4cc915..05a7a87 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -103,22 +103,26 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock int sendBufSize = Math.max(configuration.getInitialRequestBufferSize(), DEFAULT_SEND_BUFFER_SIZE); int maxSendBufSize = Math.max(configuration.getMaximumRequestBufferSize(), sendBufSize); + WebSocketSendBuffer sendBuf = null; + WebSocketSendBuffer controlBuf = null; try { - this.sendBuffer = new WebSocketSendBuffer(sendBufSize, maxSendBufSize); + sendBuf = new WebSocketSendBuffer(sendBufSize, maxSendBufSize); // Control frames (ping/pong/close) have max 125-byte payload + 14-byte header. // This dedicated buffer prevents sendPongFrame from clobbering an in-progress // frame being built in the main sendBuffer. - this.controlFrameBuffer = new WebSocketSendBuffer(256, 256); + controlBuf = new WebSocketSendBuffer(256, 256); this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE); this.maxRecvBufSize = Math.max(configuration.getMaximumResponseBufferSize(), recvBufSize); this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT); } catch (Throwable t) { - Misc.free(controlFrameBuffer); - Misc.free(sendBuffer); + Misc.free(controlBuf); + Misc.free(sendBuf); Misc.free(socket); throw t; } + this.sendBuffer = sendBuf; + this.controlFrameBuffer = controlBuf; this.recvPos = 0; this.recvReadPos = 0; @@ -284,13 +288,6 @@ public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) { } } - /** - * Receives frame with default timeout. - */ - public boolean receiveFrame(WebSocketFrameHandler handler) { - return receiveFrame(handler, defaultTimeout); - } - /** * Sends binary data as a WebSocket binary frame. * @@ -328,24 +325,6 @@ public void sendCloseFrame(int code, String reason, int timeout) { } } - /** - * Sends a complete WebSocket frame. - * - * @param frame frame info from endBinaryFrame() - * @param timeout timeout in milliseconds - */ - public void sendFrame(WebSocketSendBuffer.FrameInfo frame, int timeout) { - checkConnected(); - doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); - } - - /** - * Sends a complete WebSocket frame with default timeout. - */ - public void sendFrame(WebSocketSendBuffer.FrameInfo frame) { - sendFrame(frame, defaultTimeout); - } - /** * Sends a ping frame. */ From 0cbba889051b88bca73212ae8a8f1d98a6695d48 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 16:17:36 +0100 Subject: [PATCH 111/230] Fix receiveFrame() throwing instead of returning false receiveFrame() promises to return false on timeout, but remainingTime() threw HttpClientException when time expired. This made the return-false path dead code and caused callers like waitForAck() to abort prematurely instead of retrying. Extract remainingTime() as a pure static arithmetic helper that returns the remaining milliseconds without side effects. Add getRemainingTimeOrThrow() for send/upgrade/recvOrDie callers that need the throwing behavior. receiveFrame() now calls the non-throwing remainingTime(), so its timeout path correctly returns false as documented. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 05a7a87..7850598 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -420,7 +420,7 @@ public void upgrade(CharSequence path, int timeout) { doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout); // Read response - int remainingTimeout = remainingTime(timeout, startTime); + int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); readUpgradeResponse(remainingTimeout); upgraded = true; @@ -560,11 +560,11 @@ private void doConnect(CharSequence host, int port, int timeout) { private void doSend(long ptr, int len, int timeout) { long startTime = System.nanoTime(); while (len > 0) { - int remainingTimeout = remainingTime(timeout, startTime); + int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); ioWait(remainingTimeout, IOOperation.WRITE); int sent = dieIfNegative(socket.send(ptr, len)); while (socket.wantsTlsWrite()) { - remainingTimeout = remainingTime(timeout, startTime); + remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); ioWait(remainingTimeout, IOOperation.WRITE); dieIfNegative(socket.tlsIO(Socket.WRITE_FLAG)); } @@ -609,7 +609,7 @@ private void readUpgradeResponse(int timeout) { long startTime = System.nanoTime(); while (true) { - int remainingTimeout = remainingTime(timeout, startTime); + int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime); int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout); if (bytesRead > 0) { recvPos += bytesRead; @@ -639,7 +639,7 @@ private int recvOrDie(long ptr, int len, int timeout) { long startTime = System.nanoTime(); int n = dieIfNegative(socket.recv(ptr, len)); if (n == 0) { - ioWait(remainingTime(timeout, startTime), IOOperation.READ); + ioWait(getRemainingTimeOrThrow(timeout, startTime), IOOperation.READ); n = dieIfNegative(socket.recv(ptr, len)); } return n; @@ -666,12 +666,16 @@ private int recvOrTimeout(long ptr, int len, int timeout) { return n; } - private int remainingTime(int timeoutMillis, long startTimeNanos) { - timeoutMillis -= (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); - if (timeoutMillis <= 0) { + private int getRemainingTimeOrThrow(int timeoutMillis, long startTimeNanos) { + int remaining = remainingTime(timeoutMillis, startTimeNanos); + if (remaining <= 0) { throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']'); } - return timeoutMillis; + return remaining; + } + + private static int remainingTime(int timeoutMillis, long startTimeNanos) { + return timeoutMillis - (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); } private void resetFragmentState() { From fb7fc7552b6a16e0798e7021b00512d20128e6cf Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 16:24:26 +0100 Subject: [PATCH 112/230] Validate WebSocket payload length before cast getPayloadLength() returns a long that can be up to 2^63, but the code cast it directly to int. A malicious or buggy server sending a large-length frame caused silent truncation and potential buffer corruption. Now the client validates that payloadLength fits in an int before the cast and throws HttpClientException if it does not. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/http/client/WebSocketClient.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 7850598..e83dd5c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -725,7 +725,12 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) { if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { long payloadPtr = recvBufPtr + recvReadPos + frameParser.getHeaderSize(); - int payloadLen = (int) frameParser.getPayloadLength(); + long payloadLength = frameParser.getPayloadLength(); + if (payloadLength > Integer.MAX_VALUE) { + throw new HttpClientException("WebSocket frame payload too large [length=") + .put(payloadLength).put(']'); + } + int payloadLen = (int) payloadLength; // Unmask if needed (server frames should not be masked) if (frameParser.isMasked()) { From 42472ac8aa7ada00adf21d19409094b56d0ab0c4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 16:29:05 +0100 Subject: [PATCH 113/230] Fix buffer growth integer overflow in WebSocketClient growRecvBuffer() computes recvBufSize * 2 as an int, which overflows to negative when recvBufSize exceeds Integer.MAX_VALUE / 2. The negative result passes the newSize > maxRecvBufSize guard and reaches Unsafe.realloc with an invalid size. Promote the multiplication to long before clamping back to int. Change the guard from > to >= because Math.min already clamps newSize to maxRecvBufSize, so > would never trigger and the "already at max" error would be silently skipped. Apply the same long-promotion fix to fragmentBufSize * 2 in appendToFragmentBuffer() for consistency, even though the overflow there is currently benign due to Math.max fallback. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/cutlass/http/client/WebSocketClient.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index e83dd5c..13c04aa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -470,7 +470,7 @@ private void appendToFragmentBuffer(long payloadPtr, int payloadLen) { fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE); fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT); } else if (required > fragmentBufSize) { - int newSize = Math.min(Math.max(fragmentBufSize * 2, required), maxRecvBufSize); + int newSize = (int) Math.min(Math.max((long) fragmentBufSize * 2, required), maxRecvBufSize); fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT); fragmentBufSize = newSize; } @@ -589,8 +589,8 @@ private int findHeaderEnd() { } private void growRecvBuffer() { - int newSize = recvBufSize * 2; - if (newSize > maxRecvBufSize) { + int newSize = (int) Math.min((long) recvBufSize * 2, maxRecvBufSize); + if (newSize >= maxRecvBufSize) { if (recvBufSize >= maxRecvBufSize) { throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=") .put(recvBufSize) From 63f8f1700ad7c76ae0ddeeb91c1ac479acc212af Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 16:44:35 +0100 Subject: [PATCH 114/230] Fix array dimension product integer overflow QwpTableBuffer computes the total element count for multidimensional arrays by multiplying int dimensions together. For large arrays, this silently overflows, leading to undersized buffers and data corruption. Fix by promoting the multiplication to long arithmetic and validating the result fits in int range via checkedElementCount(), which throws LineSenderException on overflow. Apply this to all four call sites (2D and 3D for both double and long arrays). In QwpWebSocketEncoder, replace plain multiplication with Math.multiplyExact() so the encoder fails fast with ArithmeticException instead of silently wrapping around. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketEncoder.java | 4 ++-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index c62193d..8ae2c14 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -366,7 +366,7 @@ private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) for (int d = 0; d < nDims; d++) { int dimLen = shapes[shapeIdx++]; buffer.putInt(dimLen); - elemCount *= dimLen; + elemCount = Math.multiplyExact(elemCount, dimLen); } for (int e = 0; e < elemCount; e++) { @@ -390,7 +390,7 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { for (int d = 0; d < nDims; d++) { int dimLen = shapes[shapeIdx++]; buffer.putInt(dimLen); - elemCount *= dimLen; + elemCount = Math.multiplyExact(elemCount, dimLen); } for (int e = 0; e < elemCount; e++) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index a61a3f0..9e929e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -568,7 +568,8 @@ public void addDoubleArray(double[][] values) { throw new LineSenderException("irregular array shape"); } } - ensureArrayCapacity(2, dim0 * dim1); + int elemCount = checkedElementCount((long) dim0 * dim1); + ensureArrayCapacity(2, elemCount); arrayDims[valueCount] = 2; arrayShapes[arrayShapeOffset++] = dim0; arrayShapes[arrayShapeOffset++] = dim1; @@ -599,7 +600,8 @@ public void addDoubleArray(double[][][] values) { } } } - ensureArrayCapacity(3, dim0 * dim1 * dim2); + int elemCount = checkedElementCount((long) dim0 * dim1 * dim2); + ensureArrayCapacity(3, elemCount); arrayDims[valueCount] = 3; arrayShapes[arrayShapeOffset++] = dim0; arrayShapes[arrayShapeOffset++] = dim1; @@ -716,7 +718,8 @@ public void addLongArray(long[][] values) { throw new LineSenderException("irregular array shape"); } } - ensureArrayCapacity(2, dim0 * dim1); + int elemCount = checkedElementCount((long) dim0 * dim1); + ensureArrayCapacity(2, elemCount); arrayDims[valueCount] = 2; arrayShapes[arrayShapeOffset++] = dim0; arrayShapes[arrayShapeOffset++] = dim1; @@ -747,7 +750,8 @@ public void addLongArray(long[][][] values) { } } } - ensureArrayCapacity(3, dim0 * dim1 * dim2); + int elemCount = checkedElementCount((long) dim0 * dim1 * dim2); + ensureArrayCapacity(3, elemCount); arrayDims[valueCount] = 3; arrayShapes[arrayShapeOffset++] = dim0; arrayShapes[arrayShapeOffset++] = dim1; @@ -1214,6 +1218,13 @@ private void allocateStorage(byte type) { } } + private static int checkedElementCount(long product) { + if (product > Integer.MAX_VALUE) { + throw new LineSenderException("array too large: total element count exceeds int range"); + } + return (int) product; + } + private void ensureArrayCapacity(int nDims, int dataElements) { // Ensure per-row array dims capacity if (valueCount >= arrayDims.length) { From b84b690676379ae3364a37d9951a47d0b1a254da Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 2 Mar 2026 17:32:23 +0100 Subject: [PATCH 115/230] Avoid int overflow issue with array ingestion --- .../questdb/client/cutlass/qwp/client/MicrobatchBuffer.java | 6 +++--- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 0117c0f..41a3310 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -180,7 +180,7 @@ public void ensureCapacity(int requiredCapacity) { throw new IllegalStateException("Cannot resize when state is " + stateName(state)); } if (requiredCapacity > bufferCapacity) { - int newCapacity = Math.max(bufferCapacity * 2, requiredCapacity); + int newCapacity = (int) Math.min(Math.max((long) bufferCapacity * 2, requiredCapacity), Integer.MAX_VALUE); bufferPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); bufferCapacity = newCapacity; } @@ -463,7 +463,7 @@ public void write(long src, int length) { if (state != STATE_FILLING) { throw new IllegalStateException("Cannot write when state is " + stateName(state)); } - ensureCapacity(bufferPos + length); + ensureCapacity((int) Math.min((long) bufferPos + length, Integer.MAX_VALUE)); Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length); bufferPos += length; } @@ -477,7 +477,7 @@ public void writeByte(byte b) { if (state != STATE_FILLING) { throw new IllegalStateException("Cannot write when state is " + stateName(state)); } - ensureCapacity(bufferPos + 1); + ensureCapacity((int) Math.min((long) bufferPos + 1, Integer.MAX_VALUE)); Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b); bufferPos++; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c5442cd..9981c36 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1008,12 +1008,12 @@ private void addToMicrobatch(long dataPtr, int length) { // If current buffer can't hold the data, seal and swap if (activeBuffer.hasData() && - activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { + (long) activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { sealAndSwapBuffer(); } // Ensure buffer can hold the data - activeBuffer.ensureCapacity(activeBuffer.getBufferPos() + length); + activeBuffer.ensureCapacity((int) Math.min((long) activeBuffer.getBufferPos() + length, Integer.MAX_VALUE)); // Copy data to buffer activeBuffer.write(dataPtr, length); From c8e0c57f30549108a2fd7ffbcc5bc23cc81cc7c3 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 10:14:07 +0100 Subject: [PATCH 116/230] Remove unused QwpVarint class and tests QwpVarint provided standalone LEB128 varint encode/decode utilities, but no production code uses it. The actual varint encoding path goes through WebSocketSendBuffer.putVarint(), which has built-in capacity checking via ensureCapacity() on each byte write. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpVarint.java | 261 ----------------- .../cutlass/qwp/protocol/QwpVarintTest.java | 269 ------------------ 2 files changed, 530 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java deleted file mode 100644 index f02a4ca..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java +++ /dev/null @@ -1,261 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.protocol; - -import io.questdb.client.std.Unsafe; - -/** - * Variable-length integer encoding/decoding utilities for ILP v4 protocol. - * Uses unsigned LEB128 (Little Endian Base 128) encoding. - *

    - * The encoding scheme: - * - Values are split into 7-bit groups - * - Each byte uses the high bit (0x80) as a continuation flag - * - If high bit is set, more bytes follow - * - If high bit is clear, this is the last byte - *

    - * This implementation is designed for zero-allocation on hot paths. - */ -public final class QwpVarint { - - /** - * Maximum number of bytes needed to encode a 64-bit varint. - * ceil(64/7) = 10 bytes - */ - public static final int MAX_VARINT_BYTES = 10; - - /** - * Continuation bit mask - set in all bytes except the last. - */ - private static final int CONTINUATION_BIT = 0x80; - - /** - * Data mask - lower 7 bits of each byte. - */ - private static final int DATA_MASK = 0x7F; - - private QwpVarint() { - // utility class - } - - /** - * Decodes a varint from the given byte array. - * - * @param buf the buffer to read from - * @param pos the position to start reading - * @return the decoded value - * @throws IllegalArgumentException if the varint is malformed (too many bytes) - */ - public static long decode(byte[] buf, int pos) { - return decode(buf, pos, buf.length); - } - - /** - * Decodes a varint from the given byte array with bounds checking. - * - * @param buf the buffer to read from - * @param pos the position to start reading - * @param limit the maximum position to read (exclusive) - * @return the decoded value - * @throws IllegalArgumentException if the varint is malformed or buffer underflows - */ - public static long decode(byte[] buf, int pos, int limit) { - long result = 0; - int shift = 0; - int bytesRead = 0; - byte b; - - do { - if (pos >= limit) { - throw new IllegalArgumentException("incomplete varint"); - } - if (bytesRead >= MAX_VARINT_BYTES) { - throw new IllegalArgumentException("varint overflow"); - } - b = buf[pos++]; - result |= (long) (b & DATA_MASK) << shift; - shift += 7; - bytesRead++; - } while ((b & CONTINUATION_BIT) != 0); - - return result; - } - - /** - * Decodes a varint from direct memory. - * - * @param address the memory address to read from - * @param limit the maximum address to read (exclusive) - * @return the decoded value - * @throws IllegalArgumentException if the varint is malformed or buffer underflows - */ - public static long decode(long address, long limit) { - long result = 0; - int shift = 0; - int bytesRead = 0; - byte b; - - do { - if (address >= limit) { - throw new IllegalArgumentException("incomplete varint"); - } - if (bytesRead >= MAX_VARINT_BYTES) { - throw new IllegalArgumentException("varint overflow"); - } - b = Unsafe.getUnsafe().getByte(address++); - result |= (long) (b & DATA_MASK) << shift; - shift += 7; - bytesRead++; - } while ((b & CONTINUATION_BIT) != 0); - - return result; - } - - /** - * Decodes a varint from a byte array and stores both value and bytes consumed. - * - * @param buf the buffer to read from - * @param pos the position to start reading - * @param limit the maximum position to read (exclusive) - * @param result the result holder (must not be null) - * @throws IllegalArgumentException if the varint is malformed or buffer underflows - */ - public static void decode(byte[] buf, int pos, int limit, DecodeResult result) { - long value = 0; - int shift = 0; - int bytesRead = 0; - byte b; - - do { - if (pos >= limit) { - throw new IllegalArgumentException("incomplete varint"); - } - if (bytesRead >= MAX_VARINT_BYTES) { - throw new IllegalArgumentException("varint overflow"); - } - b = buf[pos++]; - value |= (long) (b & DATA_MASK) << shift; - shift += 7; - bytesRead++; - } while ((b & CONTINUATION_BIT) != 0); - - result.value = value; - result.bytesRead = bytesRead; - } - - /** - * Decodes a varint from direct memory and stores both value and bytes consumed. - * - * @param address the memory address to read from - * @param limit the maximum address to read (exclusive) - * @param result the result holder (must not be null) - * @throws IllegalArgumentException if the varint is malformed or buffer underflows - */ - public static void decode(long address, long limit, DecodeResult result) { - long value = 0; - int shift = 0; - int bytesRead = 0; - byte b; - - do { - if (address >= limit) { - throw new IllegalArgumentException("incomplete varint"); - } - if (bytesRead >= MAX_VARINT_BYTES) { - throw new IllegalArgumentException("varint overflow"); - } - b = Unsafe.getUnsafe().getByte(address++); - value |= (long) (b & DATA_MASK) << shift; - shift += 7; - bytesRead++; - } while ((b & CONTINUATION_BIT) != 0); - - result.value = value; - result.bytesRead = bytesRead; - } - - /** - * Encodes a long value as a varint to direct memory. - * - * @param address the memory address to write to - * @param value the value to encode (treated as unsigned) - * @return the new address after the encoded bytes - */ - public static long encode(long address, long value) { - while ((value & ~DATA_MASK) != 0) { - Unsafe.getUnsafe().putByte(address++, (byte) ((value & DATA_MASK) | CONTINUATION_BIT)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(address++, (byte) value); - return address; - } - - /** - * Encodes a long value as a varint into the given byte array. - * - * @param buf the buffer to write to - * @param pos the position to start writing - * @param value the value to encode (treated as unsigned) - * @return the new position after the encoded bytes - */ - public static int encode(byte[] buf, int pos, long value) { - while ((value & ~DATA_MASK) != 0) { - buf[pos++] = (byte) ((value & DATA_MASK) | CONTINUATION_BIT); - value >>>= 7; - } - buf[pos++] = (byte) value; - return pos; - } - - /** - * Calculates the number of bytes needed to encode the given value. - * - * @param value the value to measure (treated as unsigned) - * @return number of bytes needed (1-10) - */ - public static int encodedLength(long value) { - if (value == 0) { - return 1; - } - // Count leading zeros to determine the number of bits needed - int bits = 64 - Long.numberOfLeadingZeros(value); - // Each byte encodes 7 bits, round up - return (bits + 6) / 7; - } - - /** - * Result holder for decoding varints when the number of bytes consumed matters. - * This class is mutable and should be reused to avoid allocations. - */ - public static class DecodeResult { - public int bytesRead; - public long value; - - public void reset() { - value = 0; - bytesRead = 0; - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java deleted file mode 100644 index aae8ed1..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java +++ /dev/null @@ -1,269 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.protocol; - -import io.questdb.client.cutlass.qwp.protocol.QwpVarint; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Random; - -public class QwpVarintTest { - - @Test - public void testDecodeFromDirectMemory() throws Exception { - assertMemoryLeak(() -> { - long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - try { - // Encode using byte array, decode from direct memory - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 300); - - for (int i = 0; i < len; i++) { - Unsafe.getUnsafe().putByte(addr + i, buf[i]); - } - - long decoded = QwpVarint.decode(addr, addr + len); - Assert.assertEquals(300, decoded); - } finally { - Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testDecodeIncompleteVarint() { - // Byte with continuation bit set but no following byte - byte[] buf = new byte[]{(byte) 0x80}; - try { - QwpVarint.decode(buf, 0, 1); - Assert.fail("Should have thrown exception"); - } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains("incomplete varint")); - } - } - - @Test - public void testDecodeOverflow() { - // Create a buffer with too many continuation bytes (>10) - byte[] buf = new byte[12]; - for (int i = 0; i < 11; i++) { - buf[i] = (byte) 0x80; - } - buf[11] = 0x01; - - try { - QwpVarint.decode(buf, 0, 12); - Assert.fail("Should have thrown exception"); - } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains("varint overflow")); - } - } - - @Test - public void testDecodeResult() { - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 300); - - QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); - QwpVarint.decode(buf, 0, len, result); - - Assert.assertEquals(300, result.value); - Assert.assertEquals(len, result.bytesRead); - } - - @Test - public void testDecodeResultFromDirectMemory() throws Exception { - assertMemoryLeak(() -> { - long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - try { - long endAddr = QwpVarint.encode(addr, 999999); - int expectedLen = (int) (endAddr - addr); - - QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); - QwpVarint.decode(addr, endAddr, result); - - Assert.assertEquals(999999, result.value); - Assert.assertEquals(expectedLen, result.bytesRead); - } finally { - Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testDecodeResultReuse() { - byte[] buf = new byte[10]; - QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); - - // First decode - int len1 = QwpVarint.encode(buf, 0, 100); - QwpVarint.decode(buf, 0, len1, result); - Assert.assertEquals(100, result.value); - - // Reuse for second decode - result.reset(); - int len2 = QwpVarint.encode(buf, 0, 50000); - QwpVarint.decode(buf, 0, len2, result); - Assert.assertEquals(50000, result.value); - } - - @Test - public void testEncodeDecode127() { - // 127 is the maximum 1-byte value - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 127); - Assert.assertEquals(1, len); - Assert.assertEquals(0x7F, buf[0] & 0xFF); - Assert.assertEquals(127, QwpVarint.decode(buf, 0, len)); - } - - @Test - public void testEncodeDecode128() { - // 128 is the minimum 2-byte value - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 128); - Assert.assertEquals(2, len); - Assert.assertEquals(0x80, buf[0] & 0xFF); // 0 + continuation bit - Assert.assertEquals(0x01, buf[1] & 0xFF); // 1 - Assert.assertEquals(128, QwpVarint.decode(buf, 0, len)); - } - - @Test - public void testEncodeDecode16383() { - // 16383 (0x3FFF) is the maximum 2-byte value - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 16383); - Assert.assertEquals(2, len); - Assert.assertEquals(0xFF, buf[0] & 0xFF); // 127 + continuation bit - Assert.assertEquals(0x7F, buf[1] & 0xFF); // 127 - Assert.assertEquals(16383, QwpVarint.decode(buf, 0, len)); - } - - @Test - public void testEncodeDecode16384() { - // 16384 (0x4000) is the minimum 3-byte value - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 16384); - Assert.assertEquals(3, len); - Assert.assertEquals(16384, QwpVarint.decode(buf, 0, len)); - } - - @Test - public void testEncodeDecodeZero() { - byte[] buf = new byte[10]; - int len = QwpVarint.encode(buf, 0, 0); - Assert.assertEquals(1, len); - Assert.assertEquals(0x00, buf[0] & 0xFF); - Assert.assertEquals(0, QwpVarint.decode(buf, 0, len)); - } - - @Test - public void testEncodeLargeValues() { - byte[] buf = new byte[10]; - - // Test various powers of 2 - long[] values = { - 1L << 20, // ~1M - 1L << 30, // ~1B - 1L << 40, // ~1T - 1L << 50, - 1L << 60, - Long.MAX_VALUE - }; - - for (long value : values) { - int len = QwpVarint.encode(buf, 0, value); - Assert.assertTrue(len > 0 && len <= 10); - Assert.assertEquals(value, QwpVarint.decode(buf, 0, len)); - } - } - - @Test - public void testEncodeSpecificValues() { - // Test values from the spec - byte[] buf = new byte[10]; - - // 300 = 0b100101100 - // Should encode as: 0xAC (0b10101100 = 44 + 128), 0x02 (0b00000010) - int len = QwpVarint.encode(buf, 0, 300); - Assert.assertEquals(2, len); - Assert.assertEquals(0xAC, buf[0] & 0xFF); - Assert.assertEquals(0x02, buf[1] & 0xFF); - - // Verify decode - Assert.assertEquals(300, QwpVarint.decode(buf, 0, len)); - } - - @Test - public void testEncodeToDirectMemory() throws Exception { - assertMemoryLeak(() -> { - long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - try { - long endAddr = QwpVarint.encode(addr, 12345); - int len = (int) (endAddr - addr); - Assert.assertTrue(len > 0); - - // Read back and verify - long decoded = QwpVarint.decode(addr, endAddr); - Assert.assertEquals(12345, decoded); - } finally { - Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodedLength() { - Assert.assertEquals(1, QwpVarint.encodedLength(0)); - Assert.assertEquals(1, QwpVarint.encodedLength(1)); - Assert.assertEquals(1, QwpVarint.encodedLength(127)); - Assert.assertEquals(2, QwpVarint.encodedLength(128)); - Assert.assertEquals(2, QwpVarint.encodedLength(16383)); - Assert.assertEquals(3, QwpVarint.encodedLength(16384)); - // Long.MAX_VALUE = 0x7FFFFFFFFFFFFFFF (63 bits) needs ceil(63/7) = 9 bytes - Assert.assertEquals(9, QwpVarint.encodedLength(Long.MAX_VALUE)); - // Test that actual encoding matches - byte[] buf = new byte[10]; - int actualLen = QwpVarint.encode(buf, 0, Long.MAX_VALUE); - Assert.assertEquals(actualLen, QwpVarint.encodedLength(Long.MAX_VALUE)); - } - - @Test - public void testRoundTripRandomValues() { - byte[] buf = new byte[10]; - Random random = new Random(42); // Fixed seed for reproducibility - - for (int i = 0; i < 1000; i++) { - long value = random.nextLong() & Long.MAX_VALUE; // Only positive values - int len = QwpVarint.encode(buf, 0, value); - long decoded = QwpVarint.decode(buf, 0, len); - Assert.assertEquals("Failed for value: " + value, value, decoded); - } - } -} From 7f2ac45b42f24722a554562e5fd046e58d9a22e4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 10:32:52 +0100 Subject: [PATCH 117/230] Stop masking real I/O errors as timeouts recvOrTimeout() caught all HttpClientException from ioWait(), treating every error as a timeout. This silently swallowed real I/O errors such as epoll/kqueue/select failures ("queue error") and control setup failures ("epoll_ctl failure"). Add an isTimeout flag to HttpClientException and tag it via flagAsTimeout() at the two throw sites: dieWaiting() and getRemainingTimeOrThrow(). The catch in recvOrTimeout() now re-throws non-timeout exceptions instead of swallowing them. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/HttpClientException.java | 10 ++++++++++ .../client/cutlass/http/client/WebSocketClient.java | 8 +++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java b/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java index bd1f7f7..79f8185 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java @@ -31,6 +31,7 @@ public class HttpClientException extends RuntimeException { private final StringSink message = new StringSink(); private int errno = Integer.MIN_VALUE; + private boolean isTimeout; public HttpClientException(String message) { this.message.put(message); @@ -53,6 +54,10 @@ public String getMessage() { return errNoRender + " " + message; } + public boolean isTimeout() { + return isTimeout; + } + public HttpClientException put(char value) { message.put(value); return this; @@ -77,4 +82,9 @@ public HttpClientException putSize(long value) { message.putSize(value); return this; } + + public HttpClientException flagAsTimeout() { + this.isTimeout = true; + return this; + } } \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 13c04aa..375829c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -655,7 +655,9 @@ private int recvOrTimeout(long ptr, int len, int timeout) { try { ioWait(timeout, IOOperation.READ); } catch (HttpClientException e) { - // Timeout + if (!e.isTimeout()) { + throw e; + } return 0; } n = socket.recv(ptr, len); @@ -669,7 +671,7 @@ private int recvOrTimeout(long ptr, int len, int timeout) { private int getRemainingTimeOrThrow(int timeoutMillis, long startTimeNanos) { int remaining = remainingTime(timeoutMillis, startTimeNanos); if (remaining <= 0) { - throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']'); + throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']').flagAsTimeout(); } return remaining; } @@ -861,7 +863,7 @@ protected void dieWaiting(int n) { return; } if (n == 0) { - throw new HttpClientException("timed out [errno=").put(nf.errno()).put(']'); + throw new HttpClientException("timed out [errno=").put(nf.errno()).put(']').flagAsTimeout(); } throw new HttpClientException("queue error [errno=").put(nf.errno()).put(']'); } From 710537a2ff7255dd7564b631b2ccaa21c10fb149 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 10:39:33 +0100 Subject: [PATCH 118/230] Add tests for recvOrTimeout error handling Add two tests that verify recvOrTimeout() correctly distinguishes timeout exceptions from real I/O errors. A FakeSocket that always returns 0 from recv() forces the ioWait path, and a controllable ioWait override injects either a timeout or a queue error. - testRecvOrTimeoutReturnsFalseOnTimeout: verifies that a timeout-flagged exception causes receiveFrame() to return false. - testRecvOrTimeoutPropagatesNonTimeoutError: verifies that a non-timeout exception (e.g., epoll/kqueue failure) propagates out of receiveFrame() instead of being silently swallowed. Co-Authored-By: Claude Opus 4.6 --- .../http/client/WebSocketClientTest.java | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java index 647c722..aa8d142 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -27,8 +27,12 @@ import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.cutlass.http.client.HttpClientException; import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; +import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.network.Socket; +import io.questdb.client.network.TlsSessionInitFailedException; import io.questdb.client.std.Unsafe; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; @@ -99,6 +103,70 @@ public void testSendPingDoesNotClobberSendBuffer() throws Exception { }); } + @Test + public void testRecvOrTimeoutPropagatesNonTimeoutError() throws Exception { + assertMemoryLeak(() -> { + try (RecvTestWebSocketClient client = new RecvTestWebSocketClient()) { + setField(client, "upgraded", true); + + // socket.recv() returns 0, triggering the ioWait path + // ioWait throws a non-timeout error (e.g., queue/poll failure) + client.ioWaitAction = () -> { + throw new HttpClientException("queue error [errno=").put(5).put(']'); + }; + + WebSocketFrameHandler noOpHandler = new WebSocketFrameHandler() { + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + } + + @Override + public void onClose(int code, String reason) { + } + }; + + try { + client.receiveFrame(noOpHandler, 1000); + Assert.fail("expected HttpClientException for queue error"); + } catch (HttpClientException e) { + Assert.assertFalse("non-timeout error must not be flagged as timeout", e.isTimeout()); + Assert.assertTrue( + "expected queue error message, got: " + e.getMessage(), + e.getMessage().contains("queue error") + ); + } + } + }); + } + + @Test + public void testRecvOrTimeoutReturnsFalseOnTimeout() throws Exception { + assertMemoryLeak(() -> { + try (RecvTestWebSocketClient client = new RecvTestWebSocketClient()) { + setField(client, "upgraded", true); + + // socket.recv() returns 0, triggering the ioWait path + // ioWait throws a timeout error + client.ioWaitAction = () -> { + throw new HttpClientException("timed out [errno=").put(0).put(']').flagAsTimeout(); + }; + + WebSocketFrameHandler noOpHandler = new WebSocketFrameHandler() { + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + } + + @Override + public void onClose(int code, String reason) { + } + }; + + boolean result = client.receiveFrame(noOpHandler, 1000); + Assert.assertFalse("receiveFrame should return false on timeout", result); + } + }); + } + private static void setField(Object obj, String fieldName, Object value) throws Exception { Class clazz = obj.getClass(); while (clazz != null) { @@ -134,4 +202,81 @@ protected void setupIoWait() { // no-op } } + + /** + * WebSocketClient subclass with a fake socket that always returns 0 + * from recv(), forcing the ioWait path in recvOrTimeout(). + */ + private static class RecvTestWebSocketClient extends WebSocketClient { + Runnable ioWaitAction; + + RecvTestWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> new FakeSocket()); + } + + @Override + protected void ioWait(int timeout, int op) { + ioWaitAction.run(); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * Minimal Socket that always returns 0 from recv() (no data available), + * triggering the ioWait path in recvOrTimeout(). + */ + private static class FakeSocket implements Socket { + + @Override + public void close() { + } + + @Override + public int getFd() { + return 0; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void of(int fd) { + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + return 0; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + return 0; + } + + @Override + public void startTlsSession(CharSequence peerName) throws TlsSessionInitFailedException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + } } From e1dc5a32909b4ad374adaea6a6de7ec5b63a619a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 12:03:20 +0100 Subject: [PATCH 119/230] Remove unused classes and their tests Remove classes with zero production callers: - ParanoiaState.java - cairo/arr/BorrowedArray.java - cutlass/http/HttpCookie.java - cutlass/http/HttpHeaderParameterValue.java - cutlass/qwp/protocol/QwpNullBitmap.java - cutlass/qwp/protocol/QwpZigZag.java - std/AbstractLowerCaseCharSequenceHashSet.java - std/Base64Helper.java - std/ConcurrentHashMap.java - std/ConcurrentIntHashMap.java - std/FilesFacade.java - std/GenericLexer.java - std/LongObjHashMap.java - std/LowerCaseCharSequenceHashSet.java - std/ex/BytecodeException.java - std/str/DirectCharSequence.java Remove corresponding test files: - test/cutlass/qwp/protocol/QwpNullBitmapTest.java - test/cutlass/qwp/protocol/QwpZigZagTest.java - test/std/ConcurrentHashMapTest.java - test/std/ConcurrentIntHashMapTest.java Co-Authored-By: Claude Opus 4.6 --- .../java/io/questdb/client/ParanoiaState.java | 84 - .../client/cairo/arr/BorrowedArray.java | 47 - .../client/cutlass/http/HttpCookie.java | 82 - .../http/HttpHeaderParameterValue.java | 46 - .../cutlass/qwp/protocol/QwpNullBitmap.java | 310 -- .../cutlass/qwp/protocol/QwpZigZag.java | 98 - .../AbstractLowerCaseCharSequenceHashSet.java | 89 - .../io/questdb/client/std/Base64Helper.java | 116 - .../questdb/client/std/ConcurrentHashMap.java | 3791 ----------------- .../client/std/ConcurrentIntHashMap.java | 3612 ---------------- .../io/questdb/client/std/FilesFacade.java | 33 - .../io/questdb/client/std/GenericLexer.java | 435 -- .../io/questdb/client/std/LongObjHashMap.java | 110 - .../std/LowerCaseCharSequenceHashSet.java | 106 - .../client/std/ex/BytecodeException.java | 33 - .../client/std/str/DirectCharSequence.java | 33 - .../qwp/protocol/QwpNullBitmapTest.java | 316 -- .../cutlass/qwp/protocol/QwpZigZagTest.java | 166 - .../test/std/ConcurrentHashMapTest.java | 156 - .../test/std/ConcurrentIntHashMapTest.java | 114 - 20 files changed, 9777 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/ParanoiaState.java delete mode 100644 core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/http/HttpHeaderParameterValue.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java delete mode 100644 core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java delete mode 100644 core/src/main/java/io/questdb/client/std/Base64Helper.java delete mode 100644 core/src/main/java/io/questdb/client/std/ConcurrentHashMap.java delete mode 100644 core/src/main/java/io/questdb/client/std/ConcurrentIntHashMap.java delete mode 100644 core/src/main/java/io/questdb/client/std/FilesFacade.java delete mode 100644 core/src/main/java/io/questdb/client/std/GenericLexer.java delete mode 100644 core/src/main/java/io/questdb/client/std/LongObjHashMap.java delete mode 100644 core/src/main/java/io/questdb/client/std/LowerCaseCharSequenceHashSet.java delete mode 100644 core/src/main/java/io/questdb/client/std/ex/BytecodeException.java delete mode 100644 core/src/main/java/io/questdb/client/std/str/DirectCharSequence.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/std/ConcurrentHashMapTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/std/ConcurrentIntHashMapTest.java diff --git a/core/src/main/java/io/questdb/client/ParanoiaState.java b/core/src/main/java/io/questdb/client/ParanoiaState.java deleted file mode 100644 index 6141660..0000000 --- a/core/src/main/java/io/questdb/client/ParanoiaState.java +++ /dev/null @@ -1,84 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client; - -// Constants that enable various diagnostics to catch leaks, double closes, etc. -public class ParanoiaState { - /** - * 

    -     * BASIC -> validates UTF-8 in log records (throws a LogError if invalid),
-     *          throws a LogError on abandoned log records (missing .$() at the end of log statement),
-     *          detects closed stdout in LogConsoleWriter.
-     *          This introduces a low overhead to logging.
-     * AGGRESSIVE -> BASIC + holds recent history of log lines to help diagnose closed stdout,
-     *               holds the stack trace of abandoned log record.
-     *               This introduces a significant overhead to logging.
-     *
-     * When running inside JUnit/Surefire, BASIC log paranoia mode gets activated automatically.
-     * You can manually edit the code in the static { } block below to activate AGGRESSIVE instead.
-     *
-     * Logs may go silent when Maven Surefire plugin closes stdout due to broken text encoding.
-     * In BASIC mode, the log writer will detect this and print errors through System.out, which
-     * under Surefire uses an alternate channel and not stdout.
-     * In AGGRESSIVE mode, it will additionally remember the most recent log lines and print them.
-     * This will help you find the offending log line with broken encoding.
-     *
-     * The logging framework detects a common coding error where you forget to end a log statement
-     * with .$(), causing the statement not to be logged. This problem can only be detected after
-     * the fact, when you start a new log record and the previous one wasn't completed.
-     *
-     * With Log Paranoia off (LOG_PARANOIA_MODE_NONE), we only detect this problem and print an
-     * error message.
-     * In BASIC mode, we throw a LogError without a stack trace.
-     * In AGGRESSIVE mode, we capture the stack trace at every start of a log statement, so when
-     * we throw the LogError, it points to the code that created and then abandoned the log record.
-     *
    - */ - public static final int LOG_PARANOIA_MODE; - public static final int LOG_PARANOIA_MODE_AGGRESSIVE = 2; - public static final int LOG_PARANOIA_MODE_BASIC = 1; - public static final int LOG_PARANOIA_MODE_NONE = 0; - // Set to true to enable Thread Local path instances created/closed stack trace logs. - public static final boolean THREAD_LOCAL_PATH_PARANOIA_MODE = false; - // Set to true to enable stricter boundary checks on Vm memories implementations. - public static final boolean VM_PARANOIA_MODE = false; - // Set to true to enable stricter File Descriptor double close checks, trace closed usages. - public static boolean FD_PARANOIA_MODE = false; - - public static boolean isInsideJUnitTest() { - StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); - for (StackTraceElement element : stackTrace) { - String className = element.getClassName(); - if (className.startsWith("org.apache.maven.surefire") || className.startsWith("org.junit.")) { - return true; - } - } - return false; - } - - static { - LOG_PARANOIA_MODE = isInsideJUnitTest() ? LOG_PARANOIA_MODE_BASIC : LOG_PARANOIA_MODE_NONE; - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java b/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java deleted file mode 100644 index 947574a..0000000 --- a/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java +++ /dev/null @@ -1,47 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cairo.arr; - -import io.questdb.client.cairo.ColumnType; -import io.questdb.client.std.Mutable; - -public class BorrowedArray extends MutableArray implements Mutable { - - public BorrowedArray() { - this.flatView = new BorrowedFlatArrayView(); - } - - /** - * Resets to an invalid array. - */ - @Override - public void clear() { - this.type = ColumnType.UNDEFINED; - borrowedFlatView().reset(); - shape.clear(); - strides.clear(); - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java b/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java deleted file mode 100644 index b653c2d..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java +++ /dev/null @@ -1,82 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.http; - -import io.questdb.client.std.Mutable; -import io.questdb.client.std.str.CharSink; -import io.questdb.client.std.str.DirectUtf8String; -import io.questdb.client.std.str.Sinkable; -import org.jetbrains.annotations.NotNull; - -public class HttpCookie implements Mutable, Sinkable { - public DirectUtf8String cookieName; - public DirectUtf8String domain; - public long expires = -1L; - public boolean httpOnly; - public long maxAge; - public boolean partitioned; - public DirectUtf8String path; - public DirectUtf8String sameSite; - public boolean secure; - public DirectUtf8String value; - - @Override - public void clear() { - this.domain = null; - this.expires = -1L; - this.httpOnly = false; - this.maxAge = 0L; - this.partitioned = false; - this.path = null; - this.sameSite = null; - this.secure = false; - this.value = null; - this.cookieName = null; - } - - @Override - public void toSink(@NotNull CharSink sink) { - sink.put('{'); - - sink.put("cookieName=").putQuoted(cookieName); - sink.put(", value=").putQuoted(value); - if (domain != null) { - sink.put(", domain=").putQuoted(domain); - } - if (path != null) { - sink.put(", path=").putQuoted(path); - } - sink.put(", secure=").put(secure); - sink.put(", httpOnly=").put(httpOnly); - sink.put(", partitioned=").put(partitioned); - sink.put(", expires=").put(expires); - sink.put(", maxAge=").put(maxAge); - if (sameSite != null) { - sink.put(", sameSite=").putQuoted(sameSite); - } - sink.put('}'); - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cutlass/http/HttpHeaderParameterValue.java b/core/src/main/java/io/questdb/client/cutlass/http/HttpHeaderParameterValue.java deleted file mode 100644 index 980b017..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/http/HttpHeaderParameterValue.java +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.http; - -import io.questdb.client.std.str.DirectUtf8String; - -public class HttpHeaderParameterValue { - private long hi; - private DirectUtf8String str; - - public long getHi() { - return hi; - } - - public DirectUtf8String getStr() { - return str; - } - - public HttpHeaderParameterValue of(long hi, DirectUtf8String str) { - this.hi = hi; - this.str = str; - return this; - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java deleted file mode 100644 index f78f65e..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java +++ /dev/null @@ -1,310 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.protocol; - -import io.questdb.client.std.Unsafe; - -/** - * Utility class for reading and writing null bitmaps in ILP v4 format. - *

    - * Null bitmap format: - *

      - *
    • Size: ceil(rowCount / 8) bytes
      • - *
    • bit[i] = 1 means row[i] is NULL
      • - *
    • Bit order: LSB first within each byte
      • - *
    - *

    - * Example: For 10 rows where rows 0, 2, 9 are null: - * 

    - * Byte 0: 0b00000101 (bits 0,2 set)
- * Byte 1: 0b00000010 (bit 1 set, which is row 9)
- *
    - */ -public final class QwpNullBitmap { - - private QwpNullBitmap() { - // utility class - } - - /** - * Checks if all rows are null. - * - * @param address bitmap start address - * @param rowCount total number of rows - * @return true if all rows are null - */ - public static boolean allNull(long address, int rowCount) { - int fullBytes = rowCount >>> 3; - int remainingBits = rowCount & 7; - - // Check full bytes (all bits should be 1) - for (int i = 0; i < fullBytes; i++) { - byte b = Unsafe.getUnsafe().getByte(address + i); - if ((b & 0xFF) != 0xFF) { - return false; - } - } - - // Check remaining bits - if (remainingBits > 0) { - byte b = Unsafe.getUnsafe().getByte(address + fullBytes); - int mask = (1 << remainingBits) - 1; - if ((b & mask) != mask) { - return false; - } - } - - return true; - } - - /** - * Clears a row's null flag in the bitmap (direct memory). - * - * @param address bitmap start address - * @param rowIndex row index to clear - */ - public static void clearNull(long address, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - long addr = address + byteIndex; - byte b = Unsafe.getUnsafe().getByte(addr); - b &= ~(1 << bitIndex); - Unsafe.getUnsafe().putByte(addr, b); - } - - /** - * Clears a row's null flag in the bitmap (byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowIndex row index to clear - */ - public static void clearNull(byte[] bitmap, int offset, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - bitmap[offset + byteIndex] &= ~(1 << bitIndex); - } - - /** - * Counts the number of null values in the bitmap. - * - * @param address bitmap start address - * @param rowCount total number of rows - * @return count of null values - */ - public static int countNulls(long address, int rowCount) { - int count = 0; - int fullBytes = rowCount >>> 3; - int remainingBits = rowCount & 7; - - // Count full bytes - for (int i = 0; i < fullBytes; i++) { - byte b = Unsafe.getUnsafe().getByte(address + i); - count += Integer.bitCount(b & 0xFF); - } - - // Count remaining bits in last partial byte - if (remainingBits > 0) { - byte b = Unsafe.getUnsafe().getByte(address + fullBytes); - int mask = (1 << remainingBits) - 1; - count += Integer.bitCount((b & mask) & 0xFF); - } - - return count; - } - - /** - * Counts the number of null values in the bitmap (byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowCount total number of rows - * @return count of null values - */ - public static int countNulls(byte[] bitmap, int offset, int rowCount) { - int count = 0; - int fullBytes = rowCount >>> 3; - int remainingBits = rowCount & 7; - - for (int i = 0; i < fullBytes; i++) { - count += Integer.bitCount(bitmap[offset + i] & 0xFF); - } - - if (remainingBits > 0) { - byte b = bitmap[offset + fullBytes]; - int mask = (1 << remainingBits) - 1; - count += Integer.bitCount((b & mask) & 0xFF); - } - - return count; - } - - /** - * Fills the bitmap setting all rows as null (direct memory). - * - * @param address bitmap start address - * @param rowCount total number of rows - */ - public static void fillAllNull(long address, int rowCount) { - int fullBytes = rowCount >>> 3; - int remainingBits = rowCount & 7; - - // Fill full bytes with all 1s - for (int i = 0; i < fullBytes; i++) { - Unsafe.getUnsafe().putByte(address + i, (byte) 0xFF); - } - - // Set remaining bits in last byte - if (remainingBits > 0) { - byte mask = (byte) ((1 << remainingBits) - 1); - Unsafe.getUnsafe().putByte(address + fullBytes, mask); - } - } - - /** - * Clears the bitmap setting all rows as non-null (direct memory). - * - * @param address bitmap start address - * @param rowCount total number of rows - */ - public static void fillNoneNull(long address, int rowCount) { - int sizeBytes = sizeInBytes(rowCount); - for (int i = 0; i < sizeBytes; i++) { - Unsafe.getUnsafe().putByte(address + i, (byte) 0); - } - } - - /** - * Clears the bitmap setting all rows as non-null (byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowCount total number of rows - */ - public static void fillNoneNull(byte[] bitmap, int offset, int rowCount) { - int sizeBytes = sizeInBytes(rowCount); - for (int i = 0; i < sizeBytes; i++) { - bitmap[offset + i] = 0; - } - } - - /** - * Checks if a specific row is null in the bitmap (from direct memory). - * - * @param address bitmap start address - * @param rowIndex row index to check - * @return true if the row is null - */ - public static boolean isNull(long address, int rowIndex) { - int byteIndex = rowIndex >>> 3; // rowIndex / 8 - int bitIndex = rowIndex & 7; // rowIndex % 8 - byte b = Unsafe.getUnsafe().getByte(address + byteIndex); - return (b & (1 << bitIndex)) != 0; - } - - /** - * Checks if a specific row is null in the bitmap (from byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowIndex row index to check - * @return true if the row is null - */ - public static boolean isNull(byte[] bitmap, int offset, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - byte b = bitmap[offset + byteIndex]; - return (b & (1 << bitIndex)) != 0; - } - - /** - * Checks if no rows are null. - * - * @param address bitmap start address - * @param rowCount total number of rows - * @return true if no rows are null - */ - public static boolean noneNull(long address, int rowCount) { - int fullBytes = rowCount >>> 3; - int remainingBits = rowCount & 7; - - // Check full bytes - for (int i = 0; i < fullBytes; i++) { - byte b = Unsafe.getUnsafe().getByte(address + i); - if (b != 0) { - return false; - } - } - - // Check remaining bits - if (remainingBits > 0) { - byte b = Unsafe.getUnsafe().getByte(address + fullBytes); - int mask = (1 << remainingBits) - 1; - if ((b & mask) != 0) { - return false; - } - } - - return true; - } - - /** - * Sets a row as null in the bitmap (direct memory). - * - * @param address bitmap start address - * @param rowIndex row index to set as null - */ - public static void setNull(long address, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - long addr = address + byteIndex; - byte b = Unsafe.getUnsafe().getByte(addr); - b |= (1 << bitIndex); - Unsafe.getUnsafe().putByte(addr, b); - } - - /** - * Sets a row as null in the bitmap (byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowIndex row index to set as null - */ - public static void setNull(byte[] bitmap, int offset, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - bitmap[offset + byteIndex] |= (1 << bitIndex); - } - - /** - * Calculates the size in bytes needed for a null bitmap. - * - * @param rowCount number of rows - * @return bitmap size in bytes - */ - public static int sizeInBytes(long rowCount) { - return (int) ((rowCount + 7) / 8); - } -} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java deleted file mode 100644 index 512de7d..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java +++ /dev/null @@ -1,98 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.protocol; - -/** - * ZigZag encoding/decoding for signed integers. - *

    - * ZigZag encoding maps signed integers to unsigned integers so that - * numbers with small absolute value have small encoded values. - *

    - * The encoding works as follows: - * 

    - *  0 ->  0
- * -1 ->  1
- *  1 ->  2
- * -2 ->  3
- *  2 ->  4
- * ...
- *
    - *

    - * Formula: - * 

    - * encode(n) = (n << 1) ^ (n >> 63)  // for 64-bit
- * decode(n) = (n >>> 1) ^ -(n & 1)
- *
    - *

    - * This is useful when combined with varint encoding because small - * negative numbers like -1 become small positive numbers (1), which - * encode efficiently as varints. - */ -public final class QwpZigZag { - - private QwpZigZag() { - // utility class - } - - /** - * Decodes a ZigZag encoded 64-bit integer. - * - * @param value the ZigZag encoded value - * @return the original signed value - */ - public static long decode(long value) { - return (value >>> 1) ^ -(value & 1); - } - - /** - * Decodes a ZigZag encoded 32-bit integer. - * - * @param value the ZigZag encoded value - * @return the original signed value - */ - public static int decode(int value) { - return (value >>> 1) ^ -(value & 1); - } - - /** - * Encodes a signed 32-bit integer using ZigZag encoding. - * - * @param value the signed value to encode - * @return the ZigZag encoded value (unsigned interpretation) - */ - public static int encode(int value) { - return (value << 1) ^ (value >> 31); - } - - /** - * Encodes a signed 64-bit integer using ZigZag encoding. - * - * @param value the signed value to encode - * @return the ZigZag encoded value (unsigned interpretation) - */ - public static long encode(long value) { - return (value << 1) ^ (value >> 63); - } -} diff --git a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java deleted file mode 100644 index 4bb8cf9..0000000 --- a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java +++ /dev/null @@ -1,89 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import java.util.Arrays; - -public abstract class AbstractLowerCaseCharSequenceHashSet implements Mutable { - protected static final int MIN_INITIAL_CAPACITY = 16; - protected static final CharSequence noEntryKey = null; - protected final double loadFactor; - protected int capacity; - protected int free; - protected CharSequence[] keys; - protected int mask; - - public AbstractLowerCaseCharSequenceHashSet(int initialCapacity, double loadFactor) { - if (loadFactor <= 0d || loadFactor >= 1d) { - throw new IllegalArgumentException("0 < loadFactor < 1"); - } - - free = this.capacity = Math.max(initialCapacity, MIN_INITIAL_CAPACITY); - this.loadFactor = loadFactor; - keys = new CharSequence[Numbers.ceilPow2((int) (this.capacity / loadFactor))]; - mask = keys.length - 1; - } - - @Override - public void clear() { - Arrays.fill(keys, noEntryKey); - free = capacity; - } - - public boolean contains(CharSequence key) { - return keyIndex(key) < 0; - } - - public int keyIndex(CharSequence key) { - int index = Chars.lowerCaseHashCode(key) & mask; - - if (keys[index] == noEntryKey) { - return index; - } - - if (Chars.equalsIgnoreCase(key, keys[index])) { - return -index - 1; - } - - return probe(key, index); - } - - public int size() { - return capacity - free; - } - - private int probe(CharSequence key, int index) { - do { - index = (index + 1) & mask; - if (keys[index] == noEntryKey) { - return index; - } - if (Chars.equalsIgnoreCase(key, keys[index])) { - return -index - 1; - } - } while (true); - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/Base64Helper.java b/core/src/main/java/io/questdb/client/std/Base64Helper.java deleted file mode 100644 index 27d6772..0000000 --- a/core/src/main/java/io/questdb/client/std/Base64Helper.java +++ /dev/null @@ -1,116 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -// Written by Gil Tene of Azul Systems, and released to the public domain, -// as explained at http://creativecommons.org/publicdomain/zero/1.0/ -// -// @author Gil Tene - -package io.questdb.client.std; - -import java.lang.reflect.Method; - -/** - * Base64Helper exists to bridge inconsistencies in Java SE support of Base64 encoding and decoding. - * Earlier Java SE platforms (up to and including Java SE 8) supported base64 encode/decode via the - * javax.xml.bind.DatatypeConverter class, which was deprecated and eventually removed in Java SE 9. - * Later Java SE platforms (Java SE 8 and later) support base64 encode/decode via the - * java.util.Base64 class (first introduced in Java SE 8, and not available on e.g. Java SE 6 or 7). - *

    - * This makes it "hard" to write a single piece of source code that deals with base64 encodings and - * will compile and run on e.g. Java SE 7 AND Java SE 9. And such common source is a common need for - * libraries. This class is intended to encapsulate this "hard"-ness and hide the ugly pretzle-twising - * needed under the covers. - *

    - * Base64Helper provides a common API that works across Java SE 6..9 (and beyond hopefully), and - * uses late binding (Reflection) internally to avoid javac-compile-time dependencies on a specific - * Java SE version (e.g. beyond 7 or before 9). - */ -public class Base64Helper { - - private static Method decodeMethod; - // encoderObj and decoderObj are used in non-static method forms, and - // irrelevant for static method forms: - private static Object decoderObj; - private static Method encodeMethod; - private static Object encoderObj; - - /** - * Converts a Base64 encoded String to a byte array - * - * @param base64input A base64-encoded input String - * @return a byte array containing the binary representation equivalent of the Base64 encoded input - */ - public static byte[] parseBase64Binary(String base64input) { - try { - return (byte[]) decodeMethod.invoke(decoderObj, base64input); - } catch (Throwable e) { - throw new UnsupportedOperationException("Failed to use platform's base64 decode method"); - } - } - - /** - * Converts an array of bytes into a Base64 string. - * - * @param binaryArray A binary encoded input array - * @return a String containing the Base64 encoded equivalent of the binary input - */ - static String printBase64Binary(byte[] binaryArray) { - try { - return (String) encodeMethod.invoke(encoderObj, binaryArray); - } catch (Throwable e) { - throw new UnsupportedOperationException("Failed to use platform's base64 encode method"); - } - } - - static { - try { - Class javaUtilBase64Class = Class.forName("java.util.Base64"); - - Method getDecoderMethod = javaUtilBase64Class.getMethod("getDecoder"); - decoderObj = getDecoderMethod.invoke(null); - decodeMethod = decoderObj.getClass().getMethod("decode", String.class); - - Method getEncoderMethod = javaUtilBase64Class.getMethod("getEncoder"); - encoderObj = getEncoderMethod.invoke(null); - encodeMethod = encoderObj.getClass().getMethod("encodeToString", byte[].class); - } catch (Throwable e) { - decodeMethod = null; - encodeMethod = null; - } - - if (encodeMethod == null) { - decoderObj = null; - encoderObj = null; - try { - Class javaxXmlBindDatatypeConverterClass = Class.forName("javax.xml.bind.DatatypeConverter"); - decodeMethod = javaxXmlBindDatatypeConverterClass.getMethod("parseBase64Binary", String.class); - encodeMethod = javaxXmlBindDatatypeConverterClass.getMethod("printBase64Binary", byte[].class); - } catch (Throwable e) { - decodeMethod = null; - encodeMethod = null; - } - } - } -} diff --git a/core/src/main/java/io/questdb/client/std/ConcurrentHashMap.java b/core/src/main/java/io/questdb/client/std/ConcurrentHashMap.java deleted file mode 100644 index 5f2238f..0000000 --- a/core/src/main/java/io/questdb/client/std/ConcurrentHashMap.java +++ /dev/null @@ -1,3791 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -/* - * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - */ - -/* - * - * - * - * - * - * Written by Doug Lea with assistance from members of JCP JSR-166 - * Expert Group and released to the public domain, as explained at - * http://creativecommons.org/publicdomain/zero/1.0/ - */ - -import io.questdb.client.std.str.CloneableMutable; -import org.jetbrains.annotations.NotNull; - -import java.io.ObjectStreamField; -import java.io.Serializable; -import java.lang.ThreadLocal; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Hashtable; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.LockSupport; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.BiFunction; -import java.util.function.Function; - -/** - * A hash table supporting full concurrency of retrievals and - * high expected concurrency for updates. This class obeys the - * same functional specification as {@link java.util.Hashtable}, and - * includes versions of methods corresponding to each method of - * {@code Hashtable}. However, even though all operations are - * thread-safe, retrieval operations do not entail locking, - * and there is not any support for locking the entire table - * in a way that prevents all access. This class is fully - * interoperable with {@code Hashtable} in programs that rely on its - * thread safety but not on its synchronization details. - *

    Retrieval operations (including {@code get}) generally do not - * block, so may overlap with update operations (including {@code put} - * and {@code remove}). Retrievals reflect the results of the most - * recently completed update operations holding upon their - * onset. (More formally, an update operation for a given key bears a - * happens-before relation with any (non-null) retrieval for - * that key reporting the updated value.) For aggregate operations - * such as {@code putAll} and {@code clear}, concurrent retrievals may - * reflect insertion or removal of only some entries. Similarly, - * Iterators, Spliterators and Enumerations return elements reflecting the - * state of the hash table at some point at or since the creation of the - * iterator/enumeration. They do not throw {@link - * java.util.ConcurrentModificationException ConcurrentModificationException}. - * However, iterators are designed to be used by only one thread at a time. - * Bear in mind that the results of aggregate status methods including - * {@code size}, {@code isEmpty}, and {@code containsValue} are typically - * useful only when a map is not undergoing concurrent updates in other threads. - * Otherwise the results of these methods reflect transient states - * that may be adequate for monitoring or estimation purposes, but not - * for program control. - *

    The table is dynamically expanded when there are too many - * collisions (i.e., keys that have distinct hash codes but fall into - * the same slot modulo the table size), with the expected average - * effect of maintaining roughly two bins per mapping (corresponding - * to a 0.75 load factor threshold for resizing). There may be much - * variance around this average as mappings are added and removed, but - * overall, this maintains a commonly accepted time/space tradeoff for - * hash tables. However, resizing this or any other kind of hash - * table may be a relatively slow operation. When possible, it is a - * good idea to provide a size estimate as an optional {@code - * initialCapacity} constructor argument. An additional optional - * {@code loadFactor} constructor argument provides a further means of - * customizing initial table capacity by specifying the table density - * to be used in calculating the amount of space to allocate for the - * given number of elements. Also, for compatibility with previous - * versions of this class, constructors may optionally specify an - * expected {@code concurrencyLevel} as an additional hint for - * internal sizing. Note that using many keys with exactly the same - * {@code hashCode()} is a sure way to slow down performance of any - * hash table. To ameliorate impact, when keys are {@link Comparable}, - * this class may use comparison order among keys to help break ties. - *

    A {@link Set} projection of a ConcurrentHashMap may be created - * (using {@link #newKeySet()} or {@link #newKeySet(int)}), or viewed - * (using {@link #keySet(Object)} when only keys are of interest, and the - * mapped values are (perhaps transiently) not used or all take the - * same mapping value. - *

    This class and its views and iterators implement all of the - * optional methods of the {@link Map} and {@link Iterator} - * interfaces. - *

    Like {@link Hashtable} but unlike {@link HashMap}, this class - * does not allow {@code null} to be used as a key or value. - *

    ConcurrentHashMaps support a set of sequential and parallel bulk - * operations that are designed - * to be safely, and often sensibly, applied even with maps that are - * being concurrently updated by other threads; for example, when - * computing a snapshot summary of the values in a shared registry. - * There are three kinds of operation, each with four forms, accepting - * functions with Keys, Values, Entries, and (Key, Value) arguments - * and/or return values. Because the elements of a ConcurrentHashMap - * are not ordered in any particular way, and may be processed in - * different orders in different parallel executions, the correctness - * of supplied functions should not depend on any ordering, or on any - * other objects or values that may transiently change while - * computation is in progress; and except for forEach actions, should - * ideally be side-effect-free. Bulk operations on {@link java.util.Map.Entry} - * objects do not support method {@code setValue}. - *

      - *
    • forEach: Perform a given action on each element. - * A variant form applies a given transformation on each element - * before performing the action.
      • - *
    • search: Return the first available non-null result of - * applying a given function on each element; skipping further - * search when a result is found.
      • - *
    • reduce: Accumulate each element. The supplied reduction - * function cannot rely on ordering (more formally, it should be - * both associative and commutative). There are five variants: - *
        - *
      • Plain reductions. (There is not a form of this method for - * (key, value) function arguments since there is no corresponding - * return type.)
        • - *
      • Mapped reductions that accumulate the results of a given - * function applied to each element.
        • - *
      • Reductions to scalar doubles, longs, and ints, using a - * given basis value.
        • - *
      - *
      • - *
    - *

    The concurrency properties of bulk operations follow - * from those of ConcurrentHashMap: Any non-null result returned - * from {@code get(key)} and related access methods bears a - * happens-before relation with the associated insertion or - * update. The result of any bulk operation reflects the - * composition of these per-element relations (but is not - * necessarily atomic with respect to the map as a whole unless it - * is somehow known to be quiescent). Conversely, because keys - * and values in the map are never null, null serves as a reliable - * atomic indicator of the current lack of any result. To - * maintain this property, null serves as an implicit basis for - * all non-scalar reduction operations. For the double, long, and - * int versions, the basis should be one that, when combined with - * any other value, returns that other value (more formally, it - * should be the identity element for the reduction). Most common - * reductions have these properties; for example, computing a sum - * with basis 0 or a minimum with basis MAX_VALUE. - *

    Search and transformation functions provided as arguments - * should similarly return null to indicate the lack of any result - * (in which case it is not used). In the case of mapped - * reductions, this also enables transformations to serve as - * filters, returning null (or, in the case of primitive - * specializations, the identity basis) if the element should not - * be combined. You can create compound transformations and - * filterings by composing them yourself under this "null means - * there is nothing there now" rule before using them in search or - * reduce operations. - *

    Methods accepting and/or returning Entry arguments maintain - * key-value associations. They may be useful for example when - * finding the key for the greatest value. Note that "plain" Entry - * arguments can be supplied using {@code new - * AbstractMap.SimpleEntry(k,v)}. - *

    Bulk operations may complete abruptly, throwing an - * exception encountered in the application of a supplied - * function. Bear in mind when handling such exceptions that other - * concurrently executing functions could also have thrown - * exceptions, or would have done so if the first exception had - * not occurred. - *

    Speedups for parallel compared to sequential forms are common - * but not guaranteed. Parallel operations involving brief functions - * on small maps may execute more slowly than sequential forms if the - * underlying work to parallelize the computation is more expensive - * than the computation itself. Similarly, parallelization may not - * lead to much actual parallelism if all processors are busy - * performing unrelated tasks. - *

    All arguments to all task methods must be non-null. - *

    This class is a member of the - * - * Java Collections Framework. - * - * @param the type of mapped values - * @author Doug Lea - * @since 1.5 - */ -@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") -public class ConcurrentHashMap extends AbstractMap - implements ConcurrentMap, Serializable { - static final int HASH_BITS = 0x7fffffff; // usable bits of normal node hash - - /* - * Overview: - * - * The primary design goal of this hash table is to maintain - * concurrent readability (typically method get(), but also - * iterators and related methods) while minimizing update - * contention. Secondary goals are to keep space consumption about - * the same or better than java.util.HashMap, and to support high - * initial insertion rates on an empty table by many threads. - * - * This map usually acts as a binned (bucketed) hash table. Each - * key-value mapping is held in a Node. Most nodes are instances - * of the basic Node class with hash, key, value, and next - * fields. However, various subclasses exist: TreeNodes are - * arranged in balanced trees, not lists. TreeBins hold the roots - * of sets of TreeNodes. ForwardingNodes are placed at the heads - * of bins during resizing. ReservationNodes are used as - * placeholders while establishing values in computeIfAbsent and - * related methods. The types TreeBin, ForwardingNode, and - * ReservationNode do not hold normal user keys, values, or - * hashes, and are readily distinguishable during search etc - * because they have negative hash fields and null key and value - * fields. (These special nodes are either uncommon or transient, - * so the impact of carrying around some unused fields is - * insignificant.) - * - * The table is lazily initialized to a power-of-two size upon the - * first insertion. Each bin in the table normally contains a - * list of Nodes (most often, the list has only zero or one Node). - * Table accesses require volatile/atomic reads, writes, and - * CASes. Because there is no other way to arrange this without - * adding further indirections, we use intrinsics - * (sun.misc.Unsafe) operations. - * - * We use the top (sign) bit of Node hash fields for control - * purposes -- it is available anyway because of addressing - * constraints. Nodes with negative hash fields are specially - * handled or ignored in map methods. - * - * Insertion (via put or its variants) of the first node in an - * empty bin is performed by just CASing it to the bin. This is - * by far the most common case for put operations under most - * key/hash distributions. Other update operations (insert, - * delete, and replace) require locks. We do not want to waste - * the space required to associate a distinct lock object with - * each bin, so instead use the first node of a bin list itself as - * a lock. Locking support for these locks relies on builtin - * "synchronized" monitors. - * - * Using the first node of a list as a lock does not by itself - * suffice though: When a node is locked, any update must first - * validate that it is still the first node after locking it, and - * retry if not. Because new nodes are always appended to lists, - * once a node is first in a bin, it remains first until deleted - * or the bin becomes invalidated (upon resizing). - * - * The main disadvantage of per-bin locks is that other update - * operations on other nodes in a bin list protected by the same - * lock can stall, for example when user equals() or mapping - * functions take a long time. However, statistically, under - * random hash codes, this is not a common problem. Ideally, the - * frequency of nodes in bins follows a Poisson distribution - * (http://en.wikipedia.org/wiki/Poisson_distribution) with a - * parameter of about 0.5 on average, given the resizing threshold - * of 0.75, although with a large variance because of resizing - * granularity. Ignoring variance, the expected occurrences of - * list size k are (exp(-0.5) * pow(0.5, k) / factorial(k)). The - * first values are: - * - * 0: 0.60653066 - * 1: 0.30326533 - * 2: 0.07581633 - * 3: 0.01263606 - * 4: 0.00157952 - * 5: 0.00015795 - * 6: 0.00001316 - * 7: 0.00000094 - * 8: 0.00000006 - * more: less than 1 in ten million - * - * Lock contention probability for two threads accessing distinct - * elements is roughly 1 / (8 * #elements) under random hashes. - * - * Actual hash code distributions encountered in practice - * sometimes deviate significantly from uniform randomness. This - * includes the case when N > (1<<30), so some keys MUST collide. - * Similarly for dumb or hostile usages in which multiple keys are - * designed to have identical hash codes or ones that differs only - * in masked-out high bits. So we use a secondary strategy that - * applies when the number of nodes in a bin exceeds a - * threshold. These TreeBins use a balanced tree to hold nodes (a - * specialized form of red-black trees), bounding search time to - * O(log N). Each search step in a TreeBin is at least twice as - * slow as in a regular list, but given that N cannot exceed - * (1<<64) (before running out of addresses) this bounds search - * steps, lock hold times, etc, to reasonable constants (roughly - * 100 nodes inspected per operation worst case) so long as keys - * are Comparable (which is very common -- String, Long, etc). - * TreeBin nodes (TreeNodes) also maintain the same "next" - * traversal pointers as regular nodes, so can be traversed in - * iterators in the same way. - * - * The table is resized when occupancy exceeds a percentage - * threshold (nominally, 0.75, but see below). Any thread - * noticing an overfull bin may assist in resizing after the - * initiating thread allocates and sets up the replacement array. - * However, rather than stalling, these other threads may proceed - * with insertions etc. The use of TreeBins shields us from the - * worst case effects of overfilling while resizes are in - * progress. Resizing proceeds by transferring bins, one by one, - * from the table to the next table. However, threads claim small - * blocks of indices to transfer (via field transferIndex) before - * doing so, reducing contention. A generation stamp in field - * sizeCtl ensures that resizings do not overlap. Because we are - * using power-of-two expansion, the elements from each bin must - * either stay at same index, or move with a power of two - * offset. We eliminate unnecessary node creation by catching - * cases where old nodes can be reused because their next fields - * won't change. On average, only about one-sixth of them need - * cloning when a table doubles. The nodes they replace will be - * garbage collectable as soon as they are no longer referenced by - * any reader thread that may be in the midst of concurrently - * traversing table. Upon transfer, the old table bin contains - * only a special forwarding node (with hash field "MOVED") that - * contains the next table as its key. On encountering a - * forwarding node, access and update operations restart, using - * the new table. - * - * Each bin transfer requires its bin lock, which can stall - * waiting for locks while resizing. However, because other - * threads can join in and help resize rather than contend for - * locks, average aggregate waits become shorter as resizing - * progresses. The transfer operation must also ensure that all - * accessible bins in both the old and new table are usable by any - * traversal. This is arranged in part by proceeding from the - * last bin (table.length - 1) up towards the first. Upon seeing - * a forwarding node, traversals (see class Traverser) arrange to - * move to the new table without revisiting nodes. To ensure that - * no intervening nodes are skipped even when moved out of order, - * a stack (see class TableStack) is created on first encounter of - * a forwarding node during a traversal, to maintain its place if - * later processing the current table. The need for these - * save/restore mechanics is relatively rare, but when one - * forwarding node is encountered, typically many more will be. - * So Traversers use a simple caching scheme to avoid creating so - * many new TableStack nodes. (Thanks to Peter Levart for - * suggesting use of a stack here.) - * - * The traversal scheme also applies to partial traversals of - * ranges of bins (via an alternate Traverser constructor) - * to support partitioned aggregate operations. Also, read-only - * operations give up if ever forwarded to a null table, which - * provides support for shutdown-style clearing, which is also not - * currently implemented. - * - * Lazy table initialization minimizes footprint until first use, - * and also avoids resizings when the first operation is from a - * putAll, constructor with map argument, or deserialization. - * These cases attempt to override the initial capacity settings, - * but harmlessly fail to take effect in cases of races. - * - * The element count is maintained using a specialization of - * LongAdder. We need to incorporate a specialization rather than - * just use a LongAdder in order to access implicit - * contention-sensing that leads to creation of multiple - * CounterCells. The counter mechanics avoid contention on - * updates but can encounter cache thrashing if read too - * frequently during concurrent access. To avoid reading so often, - * resizing under contention is attempted only upon adding to a - * bin already holding two or more nodes. Under uniform hash - * distributions, the probability of this occurring at threshold - * is around 13%, meaning that only about 1 in 8 puts check - * threshold (and after resizing, many fewer do so). - * - * TreeBins use a special form of comparison for search and - * related operations (which is the main reason we cannot use - * existing collections such as TreeMaps). TreeBins contain - * Comparable elements, but may contain others, as well as - * elements that are Comparable but not necessarily Comparable for - * the same T, so we cannot invoke compareTo among them. To handle - * this, the tree is ordered primarily by hash value, then by - * Comparable.compareTo order if applicable. On lookup at a node, - * if elements are not comparable or compare as 0 then both left - * and right children may need to be searched in the case of tied - * hash values. (This corresponds to the full list search that - * would be necessary if all elements were non-Comparable and had - * tied hashes.) On insertion, to keep a total ordering (or as - * close as is required here) across rebalancings, we compare - * classes and identityHashCodes as tie-breakers. The red-black - * balancing code is updated from pre-jdk-collections - * (http://gee.cs.oswego.edu/dl/classes/collections/RBCell.java) - * based in turn on Cormen, Leiserson, and Rivest "Introduction to - * Algorithms" (CLR). - * - * TreeBins also require an additional locking mechanism. While - * list traversal is always possible by readers even during - * updates, tree traversal is not, mainly because of tree-rotations - * that may change the root node and/or its linkages. TreeBins - * include a simple read-write lock mechanism parasitic on the - * main bin-synchronization strategy: Structural adjustments - * associated with an insertion or removal are already bin-locked - * (and so cannot conflict with other writers) but must wait for - * ongoing readers to finish. Since there can be only one such - * waiter, we use a simple scheme using a single "waiter" field to - * block writers. However, readers need never block. If the root - * lock is held, they proceed along the slow traversal path (via - * next-pointers) until the lock becomes available or the list is - * exhausted, whichever comes first. These cases are not fast, but - * maximize aggregate expected throughput. - * - * Maintaining API and serialization compatibility with previous - * versions of this class introduces several oddities. Mainly: We - * leave untouched but unused constructor arguments referring to - * concurrencyLevel. We accept a loadFactor constructor argument, - * but apply it only to initial table capacity (which is the only - * time that we can guarantee to honor it.) We also declare an - * unused "Segment" class that is instantiated in minimal form - * only when serializing. - * - * Also, solely for compatibility with previous versions of this - * class, it extends AbstractMap, even though all of its methods - * are overridden, so it is just useless baggage. - * - * This file is organized to make things a little easier to follow - * while reading than they might otherwise: First the main static - * declarations and utilities, then fields, then main public - * methods (with a few factorings of multiple public methods into - * internal ones), then sizing methods, trees, traversers, and - * bulk operations. - */ - - /* ---------------- Constants -------------- */ - /** - * The largest possible (non-power of two) array size. - * Needed by toArray and related methods. - */ - static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; - /** - * The smallest table capacity for which bins may be treeified. - * (Otherwise the table is resized if too many nodes in a bin.) - * The value should be at least 4 * TREEIFY_THRESHOLD to avoid - * conflicts between resizing and treeification thresholds. - */ - static final int MIN_TREEIFY_CAPACITY = 64; - /* - * Encodings for Node hash fields. See above for explanation. - */ - static final int MOVED = -1; // hash for forwarding nodes - /** - * Number of CPUS, to place bounds on some sizings - */ - static final int NCPU = Runtime.getRuntime().availableProcessors(); - static final int RESERVED = -3; // hash for transient reservations - static final int TREEBIN = -2; // hash for roots of trees - /** - * The bin count threshold for using a tree rather than list for a - * bin. Bins are converted to trees when adding an element to a - * bin with at least this many nodes. The value must be greater - * than 2, and should be at least 8 to mesh with assumptions in - * tree removal about conversion back to plain bins upon - * shrinkage. - */ - static final int TREEIFY_THRESHOLD = 8; - /** - * The bin count threshold for untreeifying a (split) bin during a - * resize operation. Should be less than TREEIFY_THRESHOLD, and at - * most 6 to mesh with shrinkage detection under removal. - */ - static final int UNTREEIFY_THRESHOLD = 6; - /* ---------------- Fields -------------- */ - private static final long ABASE; - private static final int ASHIFT; - /* - * Volatile access methods are used for table elements as well as - * elements of in-progress next table while resizing. All uses of - * the tab arguments must be null checked by callers. All callers - * also paranoically precheck that tab's length is not zero (or an - * equivalent check), thus ensuring that any index argument taking - * the form of a hash value anded with (length - 1) is a valid - * index. Note that, to be correct wrt arbitrary concurrency - * errors by users, these checks must operate on local variables, - * which accounts for some odd-looking inline assignments below. - * Note that calls to setTabAt always occur within locked regions, - * and so in principle require only release ordering, not - * full volatile semantics, but are currently coded as volatile - * writes to be conservative. - */ - private static final long BASECOUNT; - private static final long CELLSBUSY; - private static final long CELLVALUE; - /** - * The default initial table capacity. Must be a power of 2 - * (i.e., at least 1) and at most MAXIMUM_CAPACITY. - */ - private static final int DEFAULT_CAPACITY = 16; - /** - * The load factor for this table. Overrides of this value in - * constructors affect only the initial table capacity. The - * actual floating point value isn't normally used -- it is - * simpler to use expressions such as {@code n - (n >>> 2)} for - * the associated resizing threshold. - */ - private static final float LOAD_FACTOR = 0.75f; - /** - * The largest possible table capacity. This value must be - * exactly 1<<30 to stay within Java array allocation and indexing - * bounds for power of two table sizes, and is further required - * because the top two bits of 32bit hash fields are used for - * control purposes. - */ - private static final int MAXIMUM_CAPACITY = 1 << 30; - /** - * Minimum number of rebinnings per transfer step. Ranges are - * subdivided to allow multiple resizer threads. This value - * serves as a lower bound to avoid resizers encountering - * excessive memory contention. The value should be at least - * DEFAULT_CAPACITY. - */ - private static final int MIN_TRANSFER_STRIDE = 16; - private static final long PROBE; - - /* ---------------- Nodes -------------- */ - /** - * The increment for generating probe values - */ - private static final int PROBE_INCREMENT = 0x9e3779b9; - - /* ---------------- Static utilities -------------- */ - /** - * The number of bits used for generation stamp in sizeCtl. - * Must be at least 6 for 32bit arrays. - */ - private static final int RESIZE_STAMP_BITS = 16; - /** - * The maximum number of threads that can help resize. - * Must fit in 32 - RESIZE_STAMP_BITS bits. - */ - private static final int MAX_RESIZERS = (1 << (32 - RESIZE_STAMP_BITS)) - 1; - /** - * The bit shift for recording size stamp in sizeCtl. - */ - private static final int RESIZE_STAMP_SHIFT = 32 - RESIZE_STAMP_BITS; - private static final long SEED; - - /* ---------------- Table element access -------------- */ - /** - * The increment of seeder per new instance - */ - private static final long SEEDER_INCREMENT = 0xbb67ae8584caa73bL; - private static final long SIZECTL; - private static final long TRANSFERINDEX; - /** - * Generates per-thread initialization/probe field - */ - private static final AtomicInteger probeGenerator = new AtomicInteger(); - /** - * The next seed for default constructors. - */ - private static final AtomicLong seeder = new AtomicLong(initialSeed()); - /** - * For serialization compatibility. - */ - private static final ObjectStreamField[] serialPersistentFields = { - new ObjectStreamField("segments", Segment[].class), - new ObjectStreamField("segmentMask", Integer.TYPE), - new ObjectStreamField("segmentShift", Integer.TYPE) - }; - private static final long serialVersionUID = 7249069246763182397L; - private final java.lang.ThreadLocal> tlTraverser = ThreadLocal.withInitial(Traverser::new); - /** - * The array of bins. Lazily initialized upon first insertion. - * Size is always a power of two. Accessed directly by iterators. - */ - transient volatile Node[] table; - /** - * Base counter value, used mainly when there is no contention, - * but also as a fallback during table initialization - * races. Updated via CAS. - */ - private transient volatile long baseCount; - /** - * Spinlock (locked via CAS) used when resizing and/or creating CounterCells. - */ - private transient volatile int cellsBusy; - /** - * Table of counter cells. When non-null, size is a power of 2. - */ - private transient volatile CounterCell[] counterCells; - // Original (since JDK1.2) Map methods - private transient EntrySetView entrySet; - private transient boolean ics = true; - /* ---------------- Public operations -------------- */ - // views - private transient KeySetView keySet; - /** - * The next table to use; non-null only while resizing. - */ - private transient volatile Node[] nextTable; - /** - * Table initialization and resizing control. When negative, the - * table is being initialized or resized: -1 for initialization, - * else -(1 + the number of active resizing threads). Otherwise, - * when table is null, holds the initial table size to use upon - * creation, or 0 for default. After initialization, holds the - * next element count value upon which to resize the table. - */ - private transient volatile int sizeCtl; - /** - * The next table index (plus one) to split while resizing. - */ - private transient volatile int transferIndex; - private transient ValuesView values; - - /** - * Creates a new, empty map with the default initial table size (16). - */ - public ConcurrentHashMap(boolean isCaseSensitive) { - this.ics = isCaseSensitive; - } - - public ConcurrentHashMap() { - this(true); - } - - /** - * Creates a new, empty map with an initial table size - * accommodating the specified number of elements without the need - * to dynamically resize. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - */ - public ConcurrentHashMap(int initialCapacity, boolean isCaseSensitive) { - this.ics = isCaseSensitive; - if (initialCapacity < 0) - throw new IllegalArgumentException(); - this.sizeCtl = ((initialCapacity >= (MAXIMUM_CAPACITY >>> 1)) ? - MAXIMUM_CAPACITY : - tableSizeFor(initialCapacity + (initialCapacity >>> 1) + 1)); - } - - public ConcurrentHashMap(int initialCapacity) { - this(initialCapacity, true); - } - - /** - * Creates a new map with the same mappings as the given map. - * - * @param m the map - */ - public ConcurrentHashMap(Map m, boolean isCaseSensitive) { - this.ics = isCaseSensitive; - this.sizeCtl = DEFAULT_CAPACITY; - putAll(m); - } - - public ConcurrentHashMap(Map m) { - this(m, true); - } - - /** - * Creates a new, empty map with an initial table size based on - * the given number of elements ({@code initialCapacity}), table - * density ({@code loadFactor}), and number of concurrently - * updating threads ({@code concurrencyLevel}). - * - * @param initialCapacity the initial capacity. The implementation - * performs internal sizing to accommodate this many elements, - * given the specified load factor. - * @param loadFactor the load factor (table density) for - * establishing the initial table size - * @throws IllegalArgumentException if the initial capacity is - * negative or the load factor or concurrencyLevel are - * nonpositive - */ - public ConcurrentHashMap(int initialCapacity, float loadFactor, boolean isCaseSensitive) { - this.ics = isCaseSensitive; - if (!(loadFactor > 0.0f) || initialCapacity < 0) - throw new IllegalArgumentException(); - if (initialCapacity < 1) // Use at least as many bins - initialCapacity = 1; // as estimated threads - long size = (long) (1.0 + (long) initialCapacity / loadFactor); - this.sizeCtl = (size >= (long) MAXIMUM_CAPACITY) ? - MAXIMUM_CAPACITY : tableSizeFor((int) size); - } - - public ConcurrentHashMap(int initialCapacity, float loadFactor) { - this(initialCapacity, loadFactor, true); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @return the new set - * @since 1.8 - */ - public static KeySetView newKeySet(boolean isCaseSensitive) { - return new KeySetView<>(new ConcurrentHashMap<>(isCaseSensitive), Boolean.TRUE); - } - - public static KeySetView newKeySet() { - return new KeySetView<>(new ConcurrentHashMap<>(), Boolean.TRUE); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @return the new set - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - * @since 1.8 - */ - public static KeySetView newKeySet(int initialCapacity, boolean isCaseSensitive) { - return new KeySetView<>(new ConcurrentHashMap<>(initialCapacity, isCaseSensitive), Boolean.TRUE); - } - - public static KeySetView newKeySet(int initialCapacity) { - return new KeySetView<>(new ConcurrentHashMap<>(initialCapacity), Boolean.TRUE); - } - - /** - * Removes all of the mappings from this map. - */ - public void clear() { - long delta = 0L; // negative number of deletions - int i = 0; - Node[] tab = table; - while (tab != null && i < tab.length) { - int fh; - Node f = tabAt(tab, i); - if (f == null) - ++i; - else if ((fh = f.hash) == MOVED) { - tab = helpTransfer(tab, f); - i = 0; // restart - } else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node p = (fh >= 0 ? f : - (f instanceof TreeBin) ? - ((TreeBin) f).first : null); - while (p != null) { - --delta; - p = p.next; - } - setTabAt(tab, i++, null); - } - } - } - } - if (delta != 0L) - addCount(delta, -1); - } - - /** - * Attempts to compute a mapping for the specified key and its - * current mapped value (or {@code null} if there is no current - * mapping). The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this Map. - * - * @param key key with which the specified value is to be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws NullPointerException if the specified key or remappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V compute(CharSequence key, BiFunction remappingFunction) { - if (key == null || remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(ics); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = remappingFunction.apply(key, null)) != null) { - delta = 1; - node = new Node<>(h, maybeCopyKey(key), val, null, ics); - } - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) { - val = remappingFunction.apply(key, null); - if (val != null) { - delta = 1; - pred.next = - new Node<>(h, maybeCopyKey(key), val, null, ics); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 1; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null) - p = r.findTreeNode(h, key, null); - else - p = null; - V pv = (p == null) ? null : p.val; - val = remappingFunction.apply(key, pv); - if (val != null) { - if (p != null) - p.val = val; - else { - delta = 1; - t.putTreeVal(h, key, val); - } - } else if (p != null) { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - break; - } - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param token token to pass to the mapping function - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws NullPointerException if the specified key or mappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(CharSequence key, Object token, BiFunction mappingFunction) { - if (key == null || mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(ics); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key, token)) != null) - node = new Node<>(h, maybeCopyKey(key), val, null, ics); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - pred.next = new Node(h, key, val, null, ics); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key, null)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws NullPointerException if the specified key or mappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(CharSequence key, Function mappingFunction) { - if (key == null || mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(ics); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key)) != null) - node = new Node<>(h, maybeCopyKey(key), val, null, ics); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key)) != null) { - added = true; - pred.next = new Node<>(h, maybeCopyKey(key), val, null, ics); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key, null)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the value for the specified key is present, attempts to - * compute a new mapping given the key and its current mapped - * value. The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this map. - * - * @param key key with which a value may be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws NullPointerException if the specified key or remappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V computeIfPresent(CharSequence key, BiFunction remappingFunction) { - if (key == null || remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == key || (keyEquals(key, ek)))) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key, null)) != null) { - val = remappingFunction.apply(key, p.val); - if (val != null) - p.val = val; - else { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (binCount != 0) - break; - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * Tests if the specified object is a key in this table. - * - * @param key possible key - * @return {@code true} if and only if the specified object - * is a key in this table, as determined by the - * {@code equals} method; {@code false} otherwise - * @throws NullPointerException if the specified key is null - */ - public boolean containsKey(Object key) { - return get(key) != null; - } - - /** - * Returns {@code true} if this map maps one or more keys to the - * specified value. Note: This method may require a full traversal - * of the map, and is much slower than method {@code containsKey}. - * - * @param value value whose presence in this map is to be tested - * @return {@code true} if this map maps one or more keys to the - * specified value - * @throws NullPointerException if the specified value is null - */ - public boolean containsValue(Object value) { - if (value == null) - throw new NullPointerException(); - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - V v; - if ((v = p.val) == value || (value.equals(v))) - return true; - } - } - return false; - } - - /** - * Returns a {@link Set} view of the mappings contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from the map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the set view - */ - @NotNull - public Set> entrySet() { - EntrySetView es; - return (es = entrySet) != null ? es : (entrySet = new EntrySetView<>(this)); - } - - /** - * Compares the specified object with this map for equality. - * Returns {@code true} if the given object is a map with the same - * mappings as this map. This operation may return misleading - * results if either map is concurrently modified during execution - * of this method. - * - * @param o object to be compared for equality with this map - * @return {@code true} if the specified object is equal to this map - */ - public boolean equals(Object o) { - if (o != this) { - if (!(o instanceof Map)) - return false; - Map m = (Map) o; - Traverser it = getTraverser(table); - for (Node p; (p = it.advance()) != null; ) { - V val = p.val; - Object v = m.get(p.key); - if (v == null || (v != val && !v.equals(val))) - return false; - } - for (Map.Entry e : m.entrySet()) { - Object mk, mv, v; - if ((mk = e.getKey()) == null || - (mv = e.getValue()) == null || - (v = get(mk)) == null || - (mv != v && !mv.equals(v))) - return false; - } - } - return true; - } - - /** - * Returns the value to which the specified key is mapped, - * or {@code null} if this map contains no mapping for the key. - *

    More formally, if this map contains a mapping from a key - * {@code k} to a value {@code v} such that {@code key.equals(k)}, - * then this method returns {@code v}; otherwise it returns - * {@code null}. (There can be at most one such mapping.) - * - * @param key map key value - * @return value to which specified key is mapped - * @throws NullPointerException if the specified key is null - */ - @Override - public V get(Object key) { - if (key instanceof CharSequence) { - return get((CharSequence) key); - } - return null; - } - - public V get(CharSequence key) { - Node[] tab; - Node e, p; - int n, eh; - CharSequence ek; - int h = spread(keyHashCode(key)); - if ((tab = table) != null && (n = tab.length) > 0 && - (e = tabAt(tab, (n - 1) & h)) != null) { - if ((eh = e.hash) == h) { - if ((ek = e.key) == key || (ek != null && keyEquals(key, ek))) - return e.val; - } else if (eh < 0) - return (p = e.find(h, key)) != null ? p.val : null; - while ((e = e.next) != null) { - if (e.hash == h && - ((ek = e.key) == key || (ek != null && keyEquals(key, ek)))) - return e.val; - } - } - return null; - } - - /** - * Returns the value to which the specified key is mapped, or the - * given default value if this map contains no mapping for the - * key. - * - * @param key the key whose associated value is to be returned - * @param defaultValue the value to return if this map contains - * no mapping for the given key - * @return the mapping for the key, if present; else the default value - * @throws NullPointerException if the specified key is null - */ - public V getOrDefault(Object key, V defaultValue) { - V v; - return (v = get(key)) == null ? defaultValue : v; - } - - // ConcurrentMap methods - - /** - * Returns the hash code value for this {@link Map}, i.e., - * the sum of, for each key-value pair in the map, - * {@code key.hashCode() ^ value.hashCode()}. - * - * @return the hash code value for this map - */ - public int hashCode() { - int h = 0; - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) - h += keyHashCode(p.key) ^ p.val.hashCode(); - } - return h; - } - - /** - * {@inheritDoc} - */ - public boolean isEmpty() { - return sumCount() <= 0L; // ignore transient negative values - } - - /** - * Returns a {@link Set} view of the keys in this map, using the - * given common mapped value for any additions (i.e., {@link - * Collection#add} and {@link Collection#addAll(Collection)}). - * This is of course only appropriate if it is acceptable to use - * the same value for all additions from this view. - * - * @param mappedValue the mapped value to use for any additions - * @return the set view - * @throws NullPointerException if the mappedValue is null - */ - public KeySetView keySet(V mappedValue) { - if (mappedValue == null) - throw new NullPointerException(); - return new KeySetView<>(this, mappedValue); - } - - /** - * Returns a {@link Set} view of the keys contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from this map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. It does not support the {@code add} or - * {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - *

    - * - * @return the set view - */ - @NotNull - public KeySetView keySet() { - KeySetView ks; - return (ks = keySet) != null ? ks : (keySet = new KeySetView<>(this, null)); - } - - // Overrides of JDK8+ Map extension method defaults - - /** - * Returns the number of mappings. This method should be used - * instead of {@link #size} because a ConcurrentHashMap may - * contain more mappings than can be represented as an int. The - * value returned is an estimate; the actual count may differ if - * there are concurrent insertions or removals. - * - * @return the number of mappings - * @since 1.8 - */ - public long mappingCount() { - return Math.max(sumCount(), 0L); // ignore transient negative values - } - - /** - * Maps the specified key to the specified value in this table. - * Neither the key nor the value can be null. - *

    The value can be retrieved by calling the {@code get} method - * with a key that is equal to the original key. - * - * @param key key with which the specified value is to be associated - * @param value value to be associated with the specified key - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key or value is null - */ - public V put(CharSequence key, V value) { - return putVal(key, value, false); - } - - /** - * Copies all of the mappings from the specified map to this one. - * These mappings replace any mappings that this map had for any of the - * keys currently in the specified map. - * - * @param m mappings to be stored in this map - */ - public void putAll(@NotNull Map m) { - tryPresize(m.size()); - for (Map.Entry e : m.entrySet()) - putVal(e.getKey(), e.getValue(), false); - } - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V putIfAbsent(@NotNull CharSequence key, V value) { - return putVal(key, value, true); - } - - /** - * {@inheritDoc} - * - * @throws NullPointerException if the specified key is null - */ - @SuppressWarnings("unchecked") - public boolean remove(@NotNull Object key, Object value) { - return value != null && replaceNode((CharSequence) key, null, (V) value) != null; - } - // Hashtable legacy methods - - /** - * Removes the key (and its corresponding value) from this map. - * This method does nothing if the key is not in the map. - * - * @param key the key that needs to be removed - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key is null - */ - public V remove(CharSequence key) { - return replaceNode(key, null, null); - } - - // ConcurrentHashMap-only methods - - /** - * {@inheritDoc} - * - * @throws NullPointerException if any of the arguments are null - */ - public boolean replace(@NotNull CharSequence key, @NotNull V oldValue, @NotNull V newValue) { - return replaceNode(key, newValue, oldValue) != null; - } - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V replace(@NotNull CharSequence key, @NotNull V value) { - return replaceNode(key, value, null); - } - - /** - * {@inheritDoc} - */ - public int size() { - long n = sumCount(); - return ((n < 0L) ? 0 : - (n > (long) Integer.MAX_VALUE) ? Integer.MAX_VALUE : - (int) n); - } - - /** - * Returns a string representation of this map. The string - * representation consists of a list of key-value mappings (in no - * particular order) enclosed in braces ("{@code {}}"). Adjacent - * mappings are separated by the characters {@code ", "} (comma - * and space). Each key-value mapping is rendered as the key - * followed by an equals sign ("{@code =}") followed by the - * associated value. - * - * @return a string representation of this map - */ - public String toString() { - Traverser it = getTraverser(table); - StringBuilder sb = new StringBuilder(); - sb.append('{'); - Node p; - if ((p = it.advance()) != null) { - for (; ; ) { - CharSequence k = p.key; - V v = p.val; - sb.append(k == this ? "(this Map)" : k); - sb.append('='); - sb.append(v == this ? "(this Map)" : v); - if ((p = it.advance()) == null) - break; - sb.append(',').append(' '); - } - } - return sb.append('}').toString(); - } - - /* ---------------- Special Nodes -------------- */ - - /** - * Returns a {@link Collection} view of the values contained in this map. - * The collection is backed by the map, so changes to the map are - * reflected in the collection, and vice-versa. The collection - * supports element removal, which removes the corresponding - * mapping from this map, via the {@code Iterator.remove}, - * {@code Collection.remove}, {@code removeAll}, - * {@code retainAll}, and {@code clear} operations. It does not - * support the {@code add} or {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the collection view - */ - @NotNull - public Collection values() { - ValuesView vs; - return (vs = values) != null ? vs : (values = new ValuesView<>(this)); - } - - /* ---------------- Table Initialization and Resizing -------------- */ - - private static long initialSeed() { - String pp = System.getProperty("java.util.secureRandomSeed"); - - if (pp != null && pp.equalsIgnoreCase("true")) { - byte[] seedBytes = java.security.SecureRandom.getSeed(8); - long s = (long) (seedBytes[0]) & 0xffL; - for (int i = 1; i < 8; ++i) - s = (s << 8) | ((long) (seedBytes[i]) & 0xffL); - return s; - } - return (mix64(System.currentTimeMillis()) ^ - mix64(System.nanoTime())); - } - - private static boolean keyEquals(final CharSequence lhs, final CharSequence rhs, boolean isCaseSensitive) { - return isCaseSensitive ? Chars.equals(lhs, rhs) : Chars.equalsIgnoreCase(lhs, rhs); - } - - private static int keyHashCode(final CharSequence key, boolean isCaseSensitive) { - return isCaseSensitive ? Chars.hashCode(key) : Chars.lowerCaseHashCode(key); - } - - private static CharSequence maybeCopyKey(CharSequence key) { - return key instanceof CloneableMutable ? ((CloneableMutable) key).copy() : key; - } - - private static long mix64(long z) { - z = (z ^ (z >>> 33)) * 0xff51afd7ed558ccdL; - z = (z ^ (z >>> 33)) * 0xc4ceb9fe1a85ec53L; - return z ^ (z >>> 33); - } - - /** - * Returns a power of two table size for the given desired capacity. - * See Hackers Delight, sec 3.2 - */ - private static int tableSizeFor(int c) { - int n = c - 1; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - return (n < 0) ? 1 : (n >= MAXIMUM_CAPACITY) ? MAXIMUM_CAPACITY : n + 1; - } - - /** - * Adds to count, and if table is too small and not already - * resizing, initiates transfer. If already resizing, helps - * perform transfer if work is available. Rechecks occupancy - * after a transfer to see if another resize is already needed - * because resizings are lagging additions. - * - * @param x the count to add - * @param check if <0, don't check resize, if <= 1 only check if uncontended - */ - private void addCount(long x, int check) { - CounterCell[] as; - long b, s; - if ((as = counterCells) != null || !Unsafe.cas(this, BASECOUNT, b = baseCount, s = b + x)) { - CounterCell a; - long v; - int m; - boolean uncontended = true; - if (as == null || (m = as.length - 1) < 0 || - (a = as[getProbe() & m]) == null || - !(uncontended = Unsafe.cas(a, CELLVALUE, v = a.value, v + x))) { - fullAddCount(x, uncontended); - return; - } - if (check <= 1) - return; - s = sumCount(); - } - if (check >= 0) { - Node[] tab, nt; - int n, sc; - while (s >= (long) (sc = sizeCtl) && (tab = table) != null && - (n = tab.length) < MAXIMUM_CAPACITY) { - int rs = resizeStamp(n); - if (sc < 0) { - if (sc >>> RESIZE_STAMP_SHIFT != rs || sc == rs + MAX_RESIZERS || (nt = nextTable) == null || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - s = sumCount(); - } - } - } - - // See LongAdder version for explanation - private void fullAddCount(long x, boolean wasUncontended) { - int h; - if ((h = getProbe()) == 0) { - localInit(); // force initialization - h = getProbe(); - wasUncontended = true; - } - boolean collide = false; // True if last slot nonempty - for (; ; ) { - CounterCell[] as; - CounterCell a; - int n; - long v; - if ((as = counterCells) != null && (n = as.length) > 0) { - if ((a = as[(n - 1) & h]) == null) { - if (cellsBusy == 0) { // Try to attach new Cell - CounterCell r = new CounterCell(x); // Optimistic create - if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean created = false; - try { // Recheck under lock - CounterCell[] rs; - int m, j; - if ((rs = counterCells) != null && - (m = rs.length) > 0 && - rs[j = (m - 1) & h] == null) { - rs[j] = r; - created = true; - } - } finally { - cellsBusy = 0; - } - if (created) - break; - continue; // Slot is now non-empty - } - } - collide = false; - } else if (!wasUncontended) // CAS already known to fail - wasUncontended = true; // Continue after rehash - else if (Unsafe.cas(a, CELLVALUE, v = a.value, v + x)) - break; - else if (counterCells != as || n >= NCPU) - collide = false; // At max size or stale - else if (!collide) - collide = true; - else if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - try { - if (counterCells == as) {// Expand table unless stale - CounterCell[] rs = new CounterCell[n << 1]; - System.arraycopy(as, 0, rs, 0, n); - counterCells = rs; - } - } finally { - cellsBusy = 0; - } - collide = false; - continue; // Retry with expanded table - } - h = advanceProbe(h); - } else if (cellsBusy == 0 && counterCells == as && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean init = false; - try { // Initialize table - if (counterCells == as) { - CounterCell[] rs = new CounterCell[2]; - rs[h & 1] = new CounterCell(x); - counterCells = rs; - init = true; - } - } finally { - cellsBusy = 0; - } - if (init) - break; - } else if (Unsafe.cas(this, BASECOUNT, v = baseCount, v + x)) - break; // Fall back on using base - } - } - - private Traverser getTraverser(Node[] tab) { - Traverser traverser = tlTraverser.get(); - int len = tab == null ? 0 : tab.length; - traverser.of(tab, len, len); - return traverser; - } - - /** - * Initializes table, using the size recorded in sizeCtl. - */ - private Node[] initTable() { - Node[] tab; - int sc; - while ((tab = table) == null || tab.length == 0) { - if ((sc = sizeCtl) < 0) - Os.pause(); // lost initialization race; just spin - else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if ((tab = table) == null || tab.length == 0) { - int n = (sc > 0) ? sc : DEFAULT_CAPACITY; - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = tab = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - break; - } - } - return tab; - } - - private boolean keyEquals(final CharSequence lhs, final CharSequence rhs) { - return keyEquals(lhs, rhs, ics); - } - /* ---------------- Counter support -------------- */ - - private int keyHashCode(final CharSequence key) { - return keyHashCode(key, ics); - } - - /** - * Moves and/or copies the nodes in each bin to new table. See - * above for explanation. - */ - private void transfer(Node[] tab, Node[] nextTab) { - int n = tab.length, stride; - if ((stride = (NCPU > 1) ? (n >>> 3) / NCPU : n) < MIN_TRANSFER_STRIDE) - stride = MIN_TRANSFER_STRIDE; // subdivide range - if (nextTab == null) { // initiating - try { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n << 1]; - nextTab = nt; - } catch (Throwable ex) { // try to cope with OOME - sizeCtl = Integer.MAX_VALUE; - return; - } - nextTable = nextTab; - transferIndex = n; - } - int nextn = nextTab.length; - ForwardingNode fwd = new ForwardingNode<>(nextTab, ics); - boolean advance = true; - boolean finishing = false; // to ensure sweep before committing nextTab - for (int i = 0, bound = 0; ; ) { - Node f; - int fh; - while (advance) { - int nextIndex, nextBound; - if (--i >= bound || finishing) - advance = false; - else if ((nextIndex = transferIndex) <= 0) { - i = -1; - advance = false; - } else if (Unsafe.getUnsafe().compareAndSwapInt - (this, TRANSFERINDEX, nextIndex, - nextBound = (nextIndex > stride ? - nextIndex - stride : 0))) { - bound = nextBound; - i = nextIndex - 1; - advance = false; - } - } - if (i < 0 || i >= n || i + n >= nextn) { - int sc; - if (finishing) { - nextTable = null; - table = nextTab; - sizeCtl = (n << 1) - (n >>> 1); - return; - } - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc = sizeCtl, sc - 1)) { - if ((sc - 2) != resizeStamp(n) << RESIZE_STAMP_SHIFT) - return; - finishing = advance = true; - i = n; // recheck before commit - } - } else if ((f = tabAt(tab, i)) == null) - advance = casTabAt(tab, i, fwd); - else if ((fh = f.hash) == MOVED) - advance = true; // already processed - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node ln, hn; - if (fh >= 0) { - int runBit = fh & n; - Node lastRun = f; - for (Node p = f.next; p != null; p = p.next) { - int b = p.hash & n; - if (b != runBit) { - runBit = b; - lastRun = p; - } - } - if (runBit == 0) { - ln = lastRun; - hn = null; - } else { - hn = lastRun; - ln = null; - } - for (Node p = f; p != lastRun; p = p.next) { - int ph = p.hash; - CharSequence pk = p.key; - V pv = p.val; - if ((ph & n) == 0) - ln = new Node<>(ph, pk, pv, ln, ics); - else - hn = new Node<>(ph, pk, pv, hn, ics); - } - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } else if (f instanceof TreeBin) { - TreeBin t = (TreeBin) f; - TreeNode lo = null, loTail = null; - TreeNode hi = null, hiTail = null; - int lc = 0, hc = 0; - for (Node e = t.first; e != null; e = e.next) { - int h = e.hash; - TreeNode p = new TreeNode<> - (h, e.key, e.val, null, null, ics); - if ((h & n) == 0) { - if ((p.prev = loTail) == null) - lo = p; - else - loTail.next = p; - loTail = p; - ++lc; - } else { - if ((p.prev = hiTail) == null) - hi = p; - else - hiTail.next = p; - hiTail = p; - ++hc; - } - } - ln = (lc <= UNTREEIFY_THRESHOLD) ? untreeify(lo) : - (hc != 0) ? new TreeBin<>(lo, ics) : t; - hn = (hc <= UNTREEIFY_THRESHOLD) ? untreeify(hi) : - (lc != 0) ? new TreeBin<>(hi, ics) : t; - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } - } - } - } - } - } - - /** - * Replaces all linked nodes in bin at given index unless table is - * too small, in which case resizes instead. - */ - private void treeifyBin(Node[] tab, int index) { - Node b; - int n; - if (tab != null) { - if ((n = tab.length) < MIN_TREEIFY_CAPACITY) - tryPresize(n << 1); - else if ((b = tabAt(tab, index)) != null && b.hash >= 0) { - synchronized (b) { - if (tabAt(tab, index) == b) { - TreeNode hd = null, tl = null; - for (Node e = b; e != null; e = e.next) { - TreeNode p = - new TreeNode<>(e.hash, e.key, e.val, null, null, ics); - if ((p.prev = tl) == null) - hd = p; - else - tl.next = p; - tl = p; - } - setTabAt(tab, index, new TreeBin<>(hd, ics)); - } - } - } - } - } - - /* ---------------- Conversion from/to TreeBins -------------- */ - - /** - * Tries to presize table to accommodate the given number of elements. - * - * @param size number of elements (doesn't need to be perfectly accurate) - */ - private void tryPresize(int size) { - int c = (size >= (MAXIMUM_CAPACITY >>> 1)) ? MAXIMUM_CAPACITY : - tableSizeFor(size + (size >>> 1) + 1); - int sc; - while ((sc = sizeCtl) >= 0) { - Node[] tab = table; - int n; - if (tab == null || (n = tab.length) == 0) { - n = Math.max(sc, c); - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if (table == tab) { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - } - } else if (c <= sc || n >= MAXIMUM_CAPACITY) - break; - else if (tab == table) { - int rs = resizeStamp(n); - if (sc < 0) { - Node[] nt; - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || (nt = nextTable) == null || - transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - } - } - } - - static int advanceProbe(int probe) { - probe ^= probe << 13; // xorshift - probe ^= probe >>> 17; - probe ^= probe << 5; - Unsafe.getUnsafe().putInt(Thread.currentThread(), PROBE, probe); - return probe; - } - - /* ---------------- TreeNodes -------------- */ - - static boolean casTabAt(Node[] tab, int i, - Node v) { - return Unsafe.getUnsafe().compareAndSwapObject(tab, ((long) i << ASHIFT) + ABASE, null, v); - } - - /* ---------------- TreeBins -------------- */ - - /** - * Returns x's Class if it is of the form "class C implements - * Comparable", else null. - */ - static Class comparableClassFor(Object x) { - if (x instanceof Comparable) { - Class c; - Type[] ts, as; - Type t; - ParameterizedType p; - if ((c = x.getClass()) == String.class) // bypass checks - return c; - if ((ts = c.getGenericInterfaces()) != null) { - for (int i = 0; i < ts.length; ++i) { - if (((t = ts[i]) instanceof ParameterizedType) && - ((p = (ParameterizedType) t).getRawType() == - Comparable.class) && - (as = p.getActualTypeArguments()) != null && - as.length == 1 && as[0] == c) // type arg is c - return c; - } - } - } - return null; - } - - /* ----------------Table Traversal -------------- */ - - /** - * Returns k.compareTo(x) if x matches kc (k's screened comparable - * class), else 0. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) // for cast to Comparable - static int compareComparables(Class kc, Object k, Object x) { - return (x == null || x.getClass() != kc ? 0 : - ((Comparable) k).compareTo(x)); - } - - static int getProbe() { - return Unsafe.getUnsafe().getInt(Thread.currentThread(), PROBE); - } - - /** - * Initialize Thread fields for the current thread. Called only - * when Thread.threadLocalRandomProbe is zero, indicating that a - * thread local seed value needs to be generated. Note that even - * though the initialization is purely thread-local, we need to - * rely on (static) atomic generators to initialize the values. - */ - static void localInit() { - int p = probeGenerator.addAndGet(PROBE_INCREMENT); - int probe = (p == 0) ? 1 : p; // skip 0 - long seed = mix64(seeder.getAndAdd(SEEDER_INCREMENT)); - Thread t = Thread.currentThread(); - Unsafe.getUnsafe().putLong(t, SEED, seed); - Unsafe.getUnsafe().putInt(t, PROBE, probe); - } - - /** - * Returns the stamp bits for resizing a table of size n. - * Must be negative when shifted left by RESIZE_STAMP_SHIFT. - */ - static int resizeStamp(int n) { - return Integer.numberOfLeadingZeros(n) | (1 << (RESIZE_STAMP_BITS - 1)); - } - - static void setTabAt(Node[] tab, int i, Node v) { - Unsafe.getUnsafe().putObjectVolatile(tab, ((long) i << ASHIFT) + ABASE, v); - } - - /** - * Spreads (XORs) higher bits of hash to lower and also forces top - * bit to 0. Because the table uses power-of-two masking, sets of - * hashes that vary only in bits above the current mask will - * always collide. (Among known examples are sets of Float keys - * holding consecutive whole numbers in small tables.) So we - * apply a transform that spreads the impact of higher bits - * downward. There is a tradeoff between speed, utility, and - * quality of bit-spreading. Because many common sets of hashes - * are already reasonably distributed (so don't benefit from - * spreading), and because we use trees to handle large sets of - * collisions in bins, we just XOR some shifted bits in the - * cheapest possible way to reduce systematic lossage, as well as - * to incorporate impact of the highest bits that would otherwise - * never be used in index calculations because of table bounds. - */ - static int spread(int h) { - return (h ^ (h >>> 16)) & HASH_BITS; - } - - @SuppressWarnings("unchecked") - static Node tabAt(Node[] tab, int i) { - return (Node) Unsafe.getUnsafe().getObjectVolatile(tab, ((long) i << ASHIFT) + ABASE); - } - - /* ----------------Views -------------- */ - - /** - * Returns a list on non-TreeNodes replacing those in given list. - */ - static Node untreeify(Node b) { - Node hd = null, tl = null; - for (Node q = b; q != null; q = q.next) { - Node p = new Node<>(q.hash, q.key, q.val, null, q.ics); - if (tl == null) - hd = p; - else - tl.next = p; - tl = p; - } - return hd; - } - - /** - * Helps transfer if a resize is in progress. - */ - final Node[] helpTransfer(Node[] tab, Node f) { - Node[] nextTab; - int sc; - if (tab != null && (f instanceof ForwardingNode) && - (nextTab = ((ForwardingNode) f).nextTable) != null) { - int rs = resizeStamp(tab.length); - while (nextTab == nextTable && table == tab && - (sc = sizeCtl) < 0) { - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) { - transfer(tab, nextTab); - break; - } - } - return nextTab; - } - return table; - } - - /** - * Implementation for put and putIfAbsent - */ - final V putVal(CharSequence key, V value, boolean onlyIfAbsent) { - if (key == null || value == null) throw new NullPointerException(); - int hash = spread(keyHashCode(key)); - int binCount = 0; - Node _new = null; - - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & hash)) == null) { - if (_new == null) { - _new = new Node<>(hash, maybeCopyKey(key), value, null, ics); - } - if (casTabAt(tab, i, _new)) { - break; // no lock when adding to empty bin - } - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - CharSequence ek; - if (e.hash == hash && - ((ek = e.key) == key || - (ek != null && keyEquals(key, ek)))) { - oldVal = e.val; - if (!onlyIfAbsent) - e.val = value; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if (_new == null) { - pred.next = new Node<>(hash, maybeCopyKey(key), value, null, ics); - } else { - pred.next = _new; - } - break; - } - } - } else if (f instanceof TreeBin) { - Node p; - binCount = 2; - if ((p = ((TreeBin) f).putTreeVal(hash, maybeCopyKey(key), value)) != null) { - oldVal = p.val; - if (!onlyIfAbsent) - p.val = value; - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (oldVal != null) - return oldVal; - break; - } - } - } - addCount(1L, binCount); - return null; - } - - /** - * Implementation for the four public remove/replace methods: - * Replaces node value with v, conditional upon match of cv if - * non-null. If resulting value is null, delete. - */ - final V replaceNode(CharSequence key, V value, V cv) { - int hash = spread(keyHashCode(key)); - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0 || - (f = tabAt(tab, i = (n - 1) & hash)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - boolean validated = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - validated = true; - for (Node e = f, pred = null; ; ) { - CharSequence ek; - if (e.hash == hash && - ((ek = e.key) == key || - (ek != null && keyEquals(key, ek)))) { - V ev = e.val; - if (cv == null || cv == ev || (cv.equals(ev))) { - oldVal = ev; - if (value != null) - e.val = value; - else if (pred != null) - pred.next = e.next; - else - setTabAt(tab, i, e.next); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - validated = true; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(hash, key, null)) != null) { - V pv = p.val; - if (cv == null || cv == pv || cv.equals(pv)) { - oldVal = pv; - if (value != null) - p.val = value; - else if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (validated) { - if (oldVal != null) { - if (value == null) - addCount(-1L, -1); - return oldVal; - } - break; - } - } - } - return null; - } - - final long sumCount() { - CounterCell[] as = counterCells; - CounterCell a; - long sum = baseCount; - if (as != null) { - for (int i = 0; i < as.length; ++i) { - if ((a = as[i]) != null) - sum += a.value; - } - } - return sum; - } - - /** - * Base of key, value, and entry Iterators. Adds fields to - * Traverser to support iterator.remove. - */ - static class BaseIterator extends Traverser { - Node lastReturned; - ConcurrentHashMap map; - - public final boolean hasNext() { - return next != null; - } - - public final void remove() { - Node p; - if ((p = lastReturned) == null) - throw new IllegalStateException(); - lastReturned = null; - map.replaceNode(p.key, null, null); - } - - void of(ConcurrentHashMap map) { - Node[] tab = map.table; - int l = tab == null ? 0 : tab.length; - super.of(tab, l, l); - this.map = map; - advance(); - } - } - - /** - * Base class for views. - */ - abstract static class CollectionView - implements Collection, java.io.Serializable { - private static final String oomeMsg = "Required array size too large"; - private static final long serialVersionUID = 7249069246763182397L; - final ConcurrentHashMap map; - - CollectionView(ConcurrentHashMap map) { - this.map = map; - } - - /** - * Removes all of the elements from this view, by removing all - * the mappings from the map backing this view. - */ - public final void clear() { - map.clear(); - } - - public abstract boolean contains(Object o); - - public final boolean containsAll(@NotNull Collection c) { - if (c != this) { - for (Object e : c) { - if (e == null || !contains(e)) - return false; - } - } - return true; - } - - /** - * Returns the map backing this view. - * - * @return the map backing this view - */ - public ConcurrentHashMap getMap() { - return map; - } - - public final boolean isEmpty() { - return map.isEmpty(); - } - - /** - * Returns an iterator over the elements in this collection. - *

    The returned iterator is - * weakly consistent. - * - * @return an iterator over the elements in this collection - */ - @NotNull - public abstract Iterator iterator(); - - public abstract boolean remove(Object o); - - public final boolean removeAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - - // implementations below rely on concrete classes supplying these - // abstract methods - - public final boolean retainAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (!c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - public final int size() { - return map.size(); - } - - @NotNull - public final Object[] toArray() { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int n = (int) sz; - Object[] r = new Object[n]; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = e; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - @NotNull - @SuppressWarnings("unchecked") - public final T[] toArray(@NotNull T[] a) { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int m = (int) sz; - T[] r = (a.length >= m) ? a : - (T[]) java.lang.reflect.Array - .newInstance(a.getClass().getComponentType(), m); - int n = r.length; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = (T) e; - } - if (a == r && i < n) { - r[i] = null; // null-terminate - return r; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - /** - * Returns a string representation of this collection. - * The string representation consists of the string representations - * of the collection's elements in the order they are returned by - * its iterator, enclosed in square brackets ({@code "[]"}). - * Adjacent elements are separated by the characters {@code ", "} - * (comma and space). Elements are converted to strings as by - * {@link String#valueOf(Object)}. - * - * @return a string representation of this collection - */ - public final String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('['); - Iterator it = iterator(); - if (it.hasNext()) { - for (; ; ) { - Object e = it.next(); - sb.append(e == this ? "(this Collection)" : e); - if (!it.hasNext()) - break; - sb.append(',').append(' '); - } - } - return sb.append(']').toString(); - } - - } - - /** - * A padded cell for distributing counts. Adapted from LongAdder - * and Striped64. See their internal docs for explanation. - */ - static final class CounterCell { - final long value; - - CounterCell(long x) { - value = x; - } - } - - static final class EntryIterator extends BaseIterator - implements Iterator> { - - public Map.Entry next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - CharSequence k = p.key; - V v = p.val; - lastReturned = p; - advance(); - return new MapEntry<>(k, v, map); - } - } - - /** - * A view of a ConcurrentHashMap as a {@link Set} of (key, value) - * entries. This class cannot be directly instantiated. See - * {@link #entrySet()}. - */ - static final class EntrySetView extends CollectionView> - implements Set>, java.io.Serializable { - private static final long serialVersionUID = 2249069246763182397L; - - private final ThreadLocal> tlEntryIterator = ThreadLocal.withInitial(EntryIterator::new); - - EntrySetView(ConcurrentHashMap map) { - super(map); - } - - public boolean add(Entry e) { - return map.putVal(e.getKey(), e.getValue(), false) == null; - } - - public boolean addAll(@NotNull Collection> c) { - boolean added = false; - for (Entry e : c) { - if (add(e)) - added = true; - } - return added; - } - - public boolean contains(Object o) { - Object k, v, r; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (r = map.get(k)) != null && - (v = e.getValue()) != null && - (v == r || v.equals(r))); - } - - public boolean equals(Object o) { - Set c; - return ((o instanceof Set) && - ((c = (Set) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - public int hashCode() { - int h = 0; - Node[] t = map.table; - if (t != null) { - Traverser it = map.getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - h += p.hashCode(); - } - } - return h; - } - - /** - * @return an iterator over the entries of the backing map - */ - @NotNull - public Iterator> iterator() { - EntryIterator it = tlEntryIterator.get(); - it.of(map); - return it; - } - - public boolean remove(Object o) { - Object k, v; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (v = e.getValue()) != null && - map.remove(k, v)); - } - } - - /** - * A node inserted at head of bins during transfer operations. - */ - static final class ForwardingNode extends Node { - final Node[] nextTable; - - ForwardingNode(Node[] tab, boolean ics) { - super(MOVED, null, null, null, ics); - this.nextTable = tab; - } - - Node find(int h, CharSequence k) { - // loop to avoid arbitrarily deep recursion on forwarding nodes - outer: - for (Node[] tab = nextTable; ; ) { - Node e; - int n; - if (k == null || tab == null || (n = tab.length) == 0 || - (e = tabAt(tab, (n - 1) & h)) == null) - return null; - for (; ; ) { - int eh; - CharSequence ek; - if ((eh = e.hash) == h && - ((ek = e.key) == k || (ek != null && keyEquals(k, ek, ics)))) - return e; - if (eh < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - continue outer; - } else - return e.find(h, k); - } - if ((e = e.next) == null) - return null; - } - } - } - } - - static final class KeyIterator extends BaseIterator - implements Iterator { - - public CharSequence next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - CharSequence k = p.key; - lastReturned = p; - advance(); - return k; - } - } - - /** - * A view of a ConcurrentHashMap as a {@link Set} of keys, in - * which additions may optionally be enabled by mapping to a - * common value. This class cannot be directly instantiated. - * See {@link #keySet() keySet()}, - * {@link #keySet(Object) keySet(V)}, - * {@link #newKeySet() newKeySet()}, - * {@link #newKeySet(int) newKeySet(int)}. - * - * @since 1.8 - */ - public static class KeySetView extends CollectionView - implements Set, java.io.Serializable { - private static final long serialVersionUID = 7249069246763182397L; - private final ThreadLocal> tlKeyIterator = ThreadLocal.withInitial(KeyIterator::new); - private final V value; - - KeySetView(ConcurrentHashMap map, V value) { // non-public - super(map); - this.value = value; - } - - /** - * Adds the specified key to this set view by mapping the key to - * the default mapped value in the backing map, if defined. - * - * @param e key to be added - * @return {@code true} if this set changed as a result of the call - * @throws NullPointerException if the specified key is null - * @throws UnsupportedOperationException if no default mapped value - * for additions was provided - */ - public boolean add(CharSequence e) { - V v; - if ((v = value) == null) - throw new UnsupportedOperationException(); - return map.putVal(e, v, true) == null; - } - - /** - * Adds all of the elements in the specified collection to this set, - * as if by calling {@link #add} on each one. - * - * @param c the elements to be inserted into this set - * @return {@code true} if this set changed as a result of the call - * @throws NullPointerException if the collection or any of its - * elements are {@code null} - * @throws UnsupportedOperationException if no default mapped value - * for additions was provided - */ - public boolean addAll(@NotNull Collection c) { - boolean added = false; - V v; - if ((v = value) == null) - throw new UnsupportedOperationException(); - for (CharSequence e : c) { - if (map.putVal(e, v, true) == null) - added = true; - } - return added; - } - - /** - * {@inheritDoc} - * - * @throws NullPointerException if the specified key is null - */ - public boolean contains(Object o) { - return map.containsKey(o); - } - - public boolean equals(Object o) { - Set c; - return ((o instanceof Set) && - ((c = (Set) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - /** - * Returns the default mapped value for additions, - * or {@code null} if additions are not supported. - * - * @return the default mapped value for additions, or {@code null} - * if not supported - */ - public V getMappedValue() { - return value; - } - - public int hashCode() { - int h = 0; - for (CharSequence e : this) - h += e.hashCode(); - return h; - } - - /** - * @return an iterator over the keys of the backing map - */ - @NotNull - public Iterator iterator() { - KeyIterator it = tlKeyIterator.get(); - it.of(map); - return it; - } - - /** - * Removes the key from this map view, by removing the key (and its - * corresponding value) from the backing map. This method does - * nothing if the key is not in the map. - * - * @param o the key to be removed from the backing map - * @return {@code true} if the backing map contained the specified key - * @throws NullPointerException if the specified key is null - */ - public boolean remove(Object o) { - return map.remove(o) != null; - } - } - - /** - * Exported Entry for EntryIterator - */ - static final class MapEntry implements Map.Entry { - final CharSequence key; // non-null - final ConcurrentHashMap map; - V val; // non-null - - MapEntry(CharSequence key, V val, ConcurrentHashMap map) { - this.key = key; - this.val = val; - this.map = map; - } - - public boolean equals(Object o) { - Object k, v; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (v = e.getValue()) != null && - (k == key || map.keyEquals((CharSequence) k, key)) && - (v == val || v.equals(val))); - } - - public CharSequence getKey() { - return key; - } - - public V getValue() { - return val; - } - - public int hashCode() { - return map.keyHashCode(key) ^ val.hashCode(); - } - - /** - * Sets our entry's value and writes through to the map. The - * value to return is somewhat arbitrary here. Since we do not - * necessarily track asynchronous changes, the most recent - * "previous" value could be different from what we return (or - * could even have been removed, in which case the put will - * re-establish). We do not and cannot guarantee more. - */ - public V setValue(V value) { - if (value == null) throw new NullPointerException(); - V v = val; - val = value; - map.put(key, value); - return v; - } - - public String toString() { - return key + "=" + val; - } - } - - /** - * Key-value entry. This class is never exported out as a - * user-mutable Map.Entry (i.e., one supporting setValue; see - * MapEntry below), but can be used for read-only traversals used - * in bulk tasks. Subclasses of Node with a negative hash field - * are special, and contain null keys and values (but are never - * exported). Otherwise, keys and vals are never null. - */ - static class Node implements Map.Entry { - final int hash; - final boolean ics; - final CharSequence key; - volatile Node next; - volatile V val; - - Node(int hash, CharSequence key, V val, Node next, boolean ics) { - this.hash = hash; - this.key = key; - this.val = val; - this.next = next; - this.ics = ics; - } - - public final boolean equals(Object o) { - Object k, v, u; - Map.Entry e; - return ((o instanceof Map.Entry) && - (k = (e = (Map.Entry) o).getKey()) != null && - (v = e.getValue()) != null && - (k == key || keyEquals((CharSequence) k, key, ics)) && - (v == (u = val) || v.equals(u))); - } - - public final CharSequence getKey() { - return key; - } - - public final V getValue() { - return val; - } - - public final int hashCode() { - return keyHashCode(key, ics) ^ val.hashCode(); - } - - public final V setValue(V value) { - throw new UnsupportedOperationException(); - } - - public final String toString() { - return key + "=" + val; - } - - /** - * Virtualized support for map.get(); overridden in subclasses. - */ - Node find(int h, CharSequence k) { - Node e = this; - if (k != null) { - do { - CharSequence ek; - if (e.hash == h && - ((ek = e.key) == k || (ek != null && keyEquals(k, ek, ics)))) - return e; - } while ((e = e.next) != null); - } - return null; - } - } - - /** - * A place-holder node used in computeIfAbsent and compute - */ - static final class ReservationNode extends Node { - ReservationNode(boolean ics) { - super(RESERVED, null, null, null, ics); - } - - Node find(int h, Object k) { - return null; - } - } - - /** - * Stripped-down version of helper class used in previous version, - * declared for the sake of serialization compatibility - */ - static class Segment extends ReentrantLock implements Serializable { - private static final long serialVersionUID = 2249069246763182397L; - final float loadFactor; - - Segment() { - this.loadFactor = ConcurrentHashMap.LOAD_FACTOR; - } - } - - /** - * Records the table, its length, and current traversal index for a - * traverser that must process a region of a forwarded table before - * proceeding with current table. - */ - static final class TableStack { - int index; - int length; - TableStack next; - Node[] tab; - } - - /** - * Encapsulates traversal for methods such as containsValue; also - * serves as a base class for other iterators and spliterators. - *

    - * Method advance visits once each still-valid node that was - * reachable upon iterator construction. It might miss some that - * were added to a bin after the bin was visited, which is OK wrt - * consistency guarantees. Maintaining this property in the face - * of possible ongoing resizes requires a fair amount of - * bookkeeping state that is difficult to optimize away amidst - * volatile accesses. Even so, traversal maintains reasonable - * throughput. - *

    - * Normally, iteration proceeds bin-by-bin traversing lists. - * However, if the table has been resized, then all future steps - * must traverse both the bin at the current index as well as at - * (index + baseSize); and so on for further resizings. To - * paranoically cope with potential sharing by users of iterators - * across threads, iteration terminates if a bounds checks fails - * for a table read. - */ - static class Traverser { - int baseIndex; // current index of initial table - int baseLimit; // index bound for initial table - int baseSize; // initial table size - int index; // index of bin to use next - Node next; // the next entry to use - TableStack stack, spare; // to save/restore on ForwardingNodes - Node[] tab; // current table; updated if resized - - /** - * Saves traversal state upon encountering a forwarding node. - */ - private void pushState(Node[] t, int i, int n) { - TableStack s = spare; // reuse if possible - if (s != null) - spare = s.next; - else - s = new TableStack<>(); - s.tab = t; - s.length = n; - s.index = i; - s.next = stack; - stack = s; - } - - /** - * Possibly pops traversal state. - * - * @param n length of current table - */ - private void recoverState(int n) { - TableStack s; - int len; - while ((s = stack) != null && (index += (len = s.length)) >= n) { - n = len; - index = s.index; - tab = s.tab; - s.tab = null; - TableStack next = s.next; - s.next = spare; // save for reuse - stack = next; - spare = s; - } - if (s == null && (index += baseSize) >= n) - index = ++baseIndex; - } - - /** - * Advances if possible, returning next valid node, or null if none. - */ - final Node advance() { - Node e; - if ((e = next) != null) - e = e.next; - for (; ; ) { - Node[] t; - int i, n; // must use locals in checks - if (e != null) - return next = e; - if (baseIndex >= baseLimit || (t = tab) == null || - (n = t.length) <= (i = index) || i < 0) - return next = null; - if ((e = tabAt(t, i)) != null && e.hash < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - e = null; - pushState(t, i, n); - continue; - } else if (e instanceof TreeBin) - e = ((TreeBin) e).first; - else - e = null; - } - if (stack != null) - recoverState(n); - else if ((index = i + baseSize) >= n) - index = ++baseIndex; // visit upper slots if present - } - } - - void of(Node[] tab, int size, int limit) { - this.tab = tab; - this.baseSize = size; - this.baseIndex = this.index = 0; - this.baseLimit = limit; - this.next = null; - } - } - - /** - * TreeNodes used at the heads of bins. TreeBins do not hold user - * keys or values, but instead point to list of TreeNodes and - * their root. They also maintain a parasitic read-write lock - * forcing writers (who hold bin lock) to wait for readers (who do - * not) to complete before tree restructuring operations. - */ - static final class TreeBin extends Node { - static final int READER = 4; // increment value for setting read lock - static final int WAITER = 2; // set when waiting for write lock - // values for lockState - static final int WRITER = 1; // set while holding write lock - private static final long LOCKSTATE; - private static final sun.misc.Unsafe U; - volatile TreeNode first; - volatile int lockState; - TreeNode root; - volatile Thread waiter; - - /** - * Creates bin with initial set of nodes headed by b. - */ - TreeBin(TreeNode b, boolean ics) { - super(TREEBIN, null, null, null, ics); - this.first = b; - TreeNode r = null; - for (TreeNode x = b, next; x != null; x = next) { - next = (TreeNode) x.next; - x.left = x.right = null; - if (r == null) { - x.parent = null; - x.red = false; - r = x; - } else { - CharSequence k = x.key; - int h = x.hash; - Class kc = null; - for (TreeNode p = r; ; ) { - int dir, ph; - CharSequence pk = p.key; - if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((kc == null && - (kc = comparableClassFor(k)) == null) || - (dir = compareComparables(kc, k, pk)) == 0) - dir = tieBreakOrder(k, pk); - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - x.parent = xp; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - r = balanceInsertion(r, x); - break; - } - } - } - } - this.root = r; - assert checkInvariants(root); - } - - /** - * Possibly blocks awaiting root lock. - */ - private void contendedLock() { - boolean waiting = false; - for (int s; ; ) { - if (((s = lockState) & ~WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, WRITER)) { - if (waiting) - waiter = null; - return; - } - } else if ((s & WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, s | WAITER)) { - waiting = true; - waiter = Thread.currentThread(); - } - } else if (waiting) - LockSupport.park(this); - } - } - - /** - * Acquires write lock for tree restructuring. - */ - private void lockRoot() { - if (!U.compareAndSwapInt(this, LOCKSTATE, 0, WRITER)) - contendedLock(); // offload to separate method - } - - /** - * Releases write lock for tree restructuring. - */ - private void unlockRoot() { - lockState = 0; - } - - static TreeNode balanceDeletion(TreeNode root, - TreeNode x) { - for (TreeNode xp, xpl, xpr; ; ) { - if (x == null || x == root) - return root; - else if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (x.red) { - x.red = false; - return root; - } else if ((xpl = xp.left) == x) { - if ((xpr = xp.right) != null && xpr.red) { - xpr.red = false; - xp.red = true; - root = rotateLeft(root, xp); - xpr = (xp = x.parent) == null ? null : xp.right; - } - if (xpr == null) - x = xp; - else { - TreeNode sl = xpr.left, sr = xpr.right; - if ((sr == null || !sr.red) && - (sl == null || !sl.red)) { - xpr.red = true; - x = xp; - } else { - if (sr == null || !sr.red) { - sl.red = false; - xpr.red = true; - root = rotateRight(root, xpr); - xpr = (xp = x.parent) == null ? - null : xp.right; - } - if (xpr != null) { - xpr.red = xp.red; - if ((sr = xpr.right) != null) - sr.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateLeft(root, xp); - } - x = root; - } - } - } else { // symmetric - if (xpl != null && xpl.red) { - xpl.red = false; - xp.red = true; - root = rotateRight(root, xp); - xpl = (xp = x.parent) == null ? null : xp.left; - } - if (xpl == null) - x = xp; - else { - TreeNode sl = xpl.left, sr = xpl.right; - if ((sl == null || !sl.red) && - (sr == null || !sr.red)) { - xpl.red = true; - x = xp; - } else { - if (sl == null || !sl.red) { - sr.red = false; - xpl.red = true; - root = rotateLeft(root, xpl); - xpl = (xp = x.parent) == null ? - null : xp.left; - } - if (xpl != null) { - xpl.red = xp.red; - if ((sl = xpl.left) != null) - sl.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateRight(root, xp); - } - x = root; - } - } - } - } - } - - static TreeNode balanceInsertion(TreeNode root, - TreeNode x) { - x.red = true; - for (TreeNode xp, xpp, xppl, xppr; ; ) { - if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (!xp.red || (xpp = xp.parent) == null) - return root; - if (xp == (xppl = xpp.left)) { - if ((xppr = xpp.right) != null && xppr.red) { - xppr.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.right) { - root = rotateLeft(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateRight(root, xpp); - } - } - } - } else { - if (xppl != null && xppl.red) { - xppl.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.left) { - root = rotateRight(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateLeft(root, xpp); - } - } - } - } - } - } - - /** - * Recursive invariant check - */ - @SuppressWarnings("SimplifiableIfStatement") - static boolean checkInvariants(TreeNode t) { - TreeNode tp = t.parent, tl = t.left, tr = t.right, - tb = t.prev, tn = (TreeNode) t.next; - if (tb != null && tb.next != t) - return false; - if (tn != null && tn.prev != t) - return false; - if (tp != null && t != tp.left && t != tp.right) - return false; - if (tl != null && (tl.parent != t || tl.hash > t.hash)) - return false; - if (tr != null && (tr.parent != t || tr.hash < t.hash)) - return false; - if (t.red && tl != null && tl.red && tr != null && tr.red) - return false; - if (tl != null && !checkInvariants(tl)) - return false; - return !(tr != null && !checkInvariants(tr)); - } - - static TreeNode rotateLeft(TreeNode root, TreeNode p) { - TreeNode r, pp, rl; - if (p != null && (r = p.right) != null) { - if ((rl = p.right = r.left) != null) - rl.parent = p; - if ((pp = r.parent = p.parent) == null) - (root = r).red = false; - else if (pp.left == p) - pp.left = r; - else - pp.right = r; - r.left = p; - p.parent = r; - } - return root; - } - - static TreeNode rotateRight(TreeNode root, TreeNode p) { - TreeNode l, pp, lr; - if (p != null && (l = p.left) != null) { - if ((lr = p.left = l.right) != null) - lr.parent = p; - if ((pp = l.parent = p.parent) == null) - (root = l).red = false; - else if (pp.right == p) - pp.right = l; - else - pp.left = l; - l.right = p; - p.parent = l; - } - return root; - } - - /** - * Tie-breaking utility for ordering insertions when equal - * hashCodes and non-comparable. We don't require a total - * order, just a consistent insertion rule to maintain - * equivalence across rebalancings. Tie-breaking further than - * necessary simplifies testing a bit. - */ - static int tieBreakOrder(Object a, Object b) { - int d; - if (a == null || b == null || - (d = a.getClass().getName(). - compareTo(b.getClass().getName())) == 0) - d = (System.identityHashCode(a) <= System.identityHashCode(b) ? - -1 : 1); - return d; - } - - /** - * Returns matching node or null if none. Tries to search - * using tree comparisons from root, but continues linear - * search when lock not available. - */ - Node find(int h, CharSequence k) { - if (k != null) { - for (Node e = first; e != null; ) { - int s; - CharSequence ek; - if (((s = lockState) & (WAITER | WRITER)) != 0) { - if (e.hash == h && - ((ek = e.key) == k || (ek != null && keyEquals(k, ek, ics)))) - return e; - e = e.next; - } else if (U.compareAndSwapInt(this, LOCKSTATE, s, - s + READER)) { - TreeNode r, p; - try { - p = ((r = root) == null ? null : - r.findTreeNode(h, k, null)); - } finally { - Thread w; - if (U.getAndAddInt(this, LOCKSTATE, -READER) == - (READER | WAITER) && (w = waiter) != null) - LockSupport.unpark(w); - } - return p; - } - } - } - return null; - } - - /** - * Finds or adds a node. - * - * @return null if added - */ - TreeNode putTreeVal(int h, CharSequence k, V v) { - Class kc = null; - boolean searched = false; - for (TreeNode p = root; ; ) { - int dir, ph; - CharSequence pk; - if (p == null) { - first = root = new TreeNode<>(h, k, v, null, null, ics); - break; - } else if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((pk = p.key) == k || (pk != null && keyEquals(k, pk, ics))) - return p; - else if ((kc == null && - (kc = comparableClassFor(k)) == null) || - (dir = compareComparables(kc, k, pk)) == 0) { - if (!searched) { - TreeNode q, ch; - searched = true; - if (((ch = p.left) != null && - (q = ch.findTreeNode(h, k, kc)) != null) || - ((ch = p.right) != null && - (q = ch.findTreeNode(h, k, kc)) != null)) - return q; - } - dir = tieBreakOrder(k, pk); - } - - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - TreeNode x, f = first; - first = x = new TreeNode<>(h, k, v, f, xp, ics); - if (f != null) - f.prev = x; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - if (!xp.red) - x.red = true; - else { - lockRoot(); - try { - root = balanceInsertion(root, x); - } finally { - unlockRoot(); - } - } - break; - } - } - assert checkInvariants(root); - return null; - } - - /** - * Removes the given node, that must be present before this - * call. This is messier than typical red-black deletion code - * because we cannot swap the contents of an interior node - * with a leaf successor that is pinned by "next" pointers - * that are accessible independently of lock. So instead we - * swap the tree linkages. - * - * @return true if now too small, so should be untreeified - */ - boolean removeTreeNode(TreeNode p) { - TreeNode next = (TreeNode) p.next; - TreeNode pred = p.prev; // unlink traversal pointers - TreeNode r, rl; - if (pred == null) - first = next; - else - pred.next = next; - if (next != null) - next.prev = pred; - if (first == null) { - root = null; - return true; - } - if ((r = root) == null || r.right == null || // too small - (rl = r.left) == null || rl.left == null) - return true; - lockRoot(); - try { - TreeNode replacement; - TreeNode pl = p.left; - TreeNode pr = p.right; - if (pl != null && pr != null) { - TreeNode s = pr, sl; - while ((sl = s.left) != null) // find successor - s = sl; - boolean c = s.red; - s.red = p.red; - p.red = c; // swap colors - TreeNode sr = s.right; - TreeNode pp = p.parent; - if (s == pr) { // p was s's direct parent - p.parent = s; - s.right = p; - } else { - TreeNode sp = s.parent; - if ((p.parent = sp) != null) { - if (s == sp.left) - sp.left = p; - else - sp.right = p; - } - s.right = pr; - pr.parent = s; - } - p.left = null; - if ((p.right = sr) != null) - sr.parent = p; - s.left = pl; - pl.parent = s; - if ((s.parent = pp) == null) - r = s; - else if (p == pp.left) - pp.left = s; - else - pp.right = s; - if (sr != null) - replacement = sr; - else - replacement = p; - } else if (pl != null) - replacement = pl; - else if (pr != null) - replacement = pr; - else - replacement = p; - if (replacement != p) { - TreeNode pp = replacement.parent = p.parent; - if (pp == null) - r = replacement; - else if (p == pp.left) - pp.left = replacement; - else - pp.right = replacement; - p.left = p.right = p.parent = null; - } - - root = (p.red) ? r : balanceDeletion(r, replacement); - - if (p == replacement) { // detach pointers - TreeNode pp; - if ((pp = p.parent) != null) { - if (p == pp.left) - pp.left = null; - else if (p == pp.right) - pp.right = null; - p.parent = null; - } - } - } finally { - unlockRoot(); - } - assert checkInvariants(root); - return false; - } - - static { - try { - U = Unsafe.getUnsafe(); - Class k = TreeBin.class; - LOCKSTATE = U.objectFieldOffset - (k.getDeclaredField("lockState")); - } catch (Exception e) { - throw new Error(e); - } - } - - - - - - - - - - - - - - - - /* ------------------------------------------------------------ */ - // Red-black tree methods, all adapted from CLR - } - - /** - * Nodes for use in TreeBins - */ - static final class TreeNode extends Node { - TreeNode left; - TreeNode parent; // red-black tree links - TreeNode prev; // needed to unlink next upon deletion - boolean red; - TreeNode right; - - TreeNode(int hash, CharSequence key, V val, Node next, - TreeNode parent, boolean ics) { - super(hash, key, val, next, ics); - this.parent = parent; - } - - Node find(int h, CharSequence k) { - return findTreeNode(h, k, null); - } - - /** - * Returns the TreeNode (or null if not found) for the given key - * starting at given root. - */ - TreeNode findTreeNode(int h, CharSequence k, Class kc) { - if (k != null) { - TreeNode p = this; - do { - int ph, dir; - CharSequence pk; - TreeNode q; - TreeNode pl = p.left, pr = p.right; - if ((ph = p.hash) > h) - p = pl; - else if (ph < h) - p = pr; - else if ((pk = p.key) == k || (pk != null && keyEquals(k, pk, ics))) - return p; - else if (pl == null) - p = pr; - else if (pr == null) - p = pl; - else if ((kc != null || - (kc = comparableClassFor(k)) != null) && - (dir = compareComparables(kc, k, pk)) != 0) - p = (dir < 0) ? pl : pr; - else if ((q = pr.findTreeNode(h, k, kc)) != null) - return q; - else - p = pl; - } while (p != null); - } - return null; - } - - - } - - static final class ValueIterator extends BaseIterator - implements Iterator { - public V next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - V v = p.val; - lastReturned = p; - advance(); - return v; - } - } - - /** - * A view of a ConcurrentHashMap as a {@link Collection} of - * values, in which additions are disabled. This class cannot be - * directly instantiated. See {@link #values()}. - */ - static final class ValuesView extends CollectionView - implements Collection, java.io.Serializable { - private static final long serialVersionUID = 2249069246763182397L; - private final ThreadLocal> tlValueIterator = ThreadLocal.withInitial(ValueIterator::new); - - ValuesView(ConcurrentHashMap map) { - super(map); - } - - public boolean add(V e) { - throw new UnsupportedOperationException(); - } - - public boolean addAll(@NotNull Collection c) { - throw new UnsupportedOperationException(); - } - - public boolean contains(Object o) { - return map.containsValue(o); - } - - @NotNull - public Iterator iterator() { - ValueIterator it = tlValueIterator.get(); - it.of(map); - return it; - } - - public boolean remove(Object o) { - if (o != null) { - for (Iterator it = iterator(); it.hasNext(); ) { - if (o.equals(it.next())) { - it.remove(); - return true; - } - } - } - return false; - } - } - - static { - try { - Class tk = Thread.class; - SEED = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomSeed")); - PROBE = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomProbe")); - } catch (Exception e) { - throw new Error(e); - } - } - - static { - try { - Class k = ConcurrentHashMap.class; - SIZECTL = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("sizeCtl")); - TRANSFERINDEX = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("transferIndex")); - BASECOUNT = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("baseCount")); - CELLSBUSY = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("cellsBusy")); - Class ck = CounterCell.class; - CELLVALUE = Unsafe.getUnsafe().objectFieldOffset - (ck.getDeclaredField("value")); - Class ak = Node[].class; - ABASE = Unsafe.getUnsafe().arrayBaseOffset(ak); - int scale = Unsafe.getUnsafe().arrayIndexScale(ak); - if ((scale & (scale - 1)) != 0) - throw new Error("data type scale not a power of two"); - ASHIFT = 31 - Integer.numberOfLeadingZeros(scale); - } catch (Exception e) { - throw new Error(e); - } - } -} diff --git a/core/src/main/java/io/questdb/client/std/ConcurrentIntHashMap.java b/core/src/main/java/io/questdb/client/std/ConcurrentIntHashMap.java deleted file mode 100644 index f4dffe3..0000000 --- a/core/src/main/java/io/questdb/client/std/ConcurrentIntHashMap.java +++ /dev/null @@ -1,3612 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -/* - * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - */ - -/* - * - * - * - * - * - * Written by Doug Lea with assistance from members of JCP JSR-166 - * Expert Group and released to the public domain, as explained at - * http://creativecommons.org/publicdomain/zero/1.0/ - */ - -import org.jetbrains.annotations.NotNull; - -import java.io.ObjectStreamField; -import java.io.Serializable; -import java.lang.ThreadLocal; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.LockSupport; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.IntFunction; - -/** - * Same as {@link ConcurrentHashMap}, but with primitive type int keys. - */ -@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") -public class ConcurrentIntHashMap implements Serializable { - static final int EMPTY_KEY = Integer.MIN_VALUE; - - /* - * Overview: - * - * The primary design goal of this hash table is to maintain - * concurrent readability (typically method get(), but also - * iterators and related methods) while minimizing update - * contention. Secondary goals are to keep space consumption about - * the same or better than java.util.HashMap, and to support high - * initial insertion rates on an empty table by many threads. - * - * This map usually acts as a binned (bucketed) hash table. Each - * key-value mapping is held in a Node. Most nodes are instances - * of the basic Node class with hash, key, value, and next - * fields. However, various subclasses exist: TreeNodes are - * arranged in balanced trees, not lists. TreeBins hold the roots - * of sets of TreeNodes. ForwardingNodes are placed at the heads - * of bins during resizing. ReservationNodes are used as - * placeholders while establishing values in computeIfAbsent and - * related methods. The types TreeBin, ForwardingNode, and - * ReservationNode do not hold normal user keys, values, or - * hashes, and are readily distinguishable during search etc - * because they have negative hash fields and null key and value - * fields. (These special nodes are either uncommon or transient, - * so the impact of carrying around some unused fields is - * insignificant.) - * - * The table is lazily initialized to a power-of-two size upon the - * first insertion. Each bin in the table normally contains a - * list of Nodes (most often, the list has only zero or one Node). - * Table accesses require volatile/atomic reads, writes, and - * CASes. Because there is no other way to arrange this without - * adding further indirections, we use intrinsics - * (sun.misc.Unsafe) operations. - * - * We use the top (sign) bit of Node hash fields for control - * purposes -- it is available anyway because of addressing - * constraints. Nodes with negative hash fields are specially - * handled or ignored in map methods. - * - * Insertion (via put or its variants) of the first node in an - * empty bin is performed by just CASing it to the bin. This is - * by far the most common case for put operations under most - * key/hash distributions. Other update operations (insert, - * delete, and replace) require locks. We do not want to waste - * the space required to associate a distinct lock object with - * each bin, so instead use the first node of a bin list itself as - * a lock. Locking support for these locks relies on builtin - * "synchronized" monitors. - * - * Using the first node of a list as a lock does not by itself - * suffice though: When a node is locked, any update must first - * validate that it is still the first node after locking it, and - * retry if not. Because new nodes are always appended to lists, - * once a node is first in a bin, it remains first until deleted - * or the bin becomes invalidated (upon resizing). - * - * The main disadvantage of per-bin locks is that other update - * operations on other nodes in a bin list protected by the same - * lock can stall, for example when user equals() or mapping - * functions take a long time. However, statistically, under - * random hash codes, this is not a common problem. Ideally, the - * frequency of nodes in bins follows a Poisson distribution - * (http://en.wikipedia.org/wiki/Poisson_distribution) with a - * parameter of about 0.5 on average, given the resizing threshold - * of 0.75, although with a large variance because of resizing - * granularity. Ignoring variance, the expected occurrences of - * list size k are (exp(-0.5) * pow(0.5, k) / factorial(k)). The - * first values are: - * - * 0: 0.60653066 - * 1: 0.30326533 - * 2: 0.07581633 - * 3: 0.01263606 - * 4: 0.00157952 - * 5: 0.00015795 - * 6: 0.00001316 - * 7: 0.00000094 - * 8: 0.00000006 - * more: less than 1 in ten million - * - * Lock contention probability for two threads accessing distinct - * elements is roughly 1 / (8 * #elements) under random hashes. - * - * Actual hash code distributions encountered in practice - * sometimes deviate significantly from uniform randomness. This - * includes the case when N > (1<<30), so some keys MUST collide. - * Similarly for dumb or hostile usages in which multiple keys are - * designed to have identical hash codes or ones that differs only - * in masked-out high bits. So we use a secondary strategy that - * applies when the number of nodes in a bin exceeds a - * threshold. These TreeBins use a balanced tree to hold nodes (a - * specialized form of red-black trees), bounding search time to - * O(log N). Each search step in a TreeBin is at least twice as - * slow as in a regular list, but given that N cannot exceed - * (1<<64) (before running out of addresses) this bounds search - * steps, lock hold times, etc, to reasonable constants (roughly - * 100 nodes inspected per operation worst case) so long as keys - * are Comparable (which is very common -- String, Long, etc). - * TreeBin nodes (TreeNodes) also maintain the same "next" - * traversal pointers as regular nodes, so can be traversed in - * iterators in the same way. - * - * The table is resized when occupancy exceeds a percentage - * threshold (nominally, 0.75, but see below). Any thread - * noticing an overfull bin may assist in resizing after the - * initiating thread allocates and sets up the replacement array. - * However, rather than stalling, these other threads may proceed - * with insertions etc. The use of TreeBins shields us from the - * worst case effects of overfilling while resizes are in - * progress. Resizing proceeds by transferring bins, one by one, - * from the table to the next table. However, threads claim small - * blocks of indices to transfer (via field transferIndex) before - * doing so, reducing contention. A generation stamp in field - * sizeCtl ensures that resizings do not overlap. Because we are - * using power-of-two expansion, the elements from each bin must - * either stay at same index, or move with a power of two - * offset. We eliminate unnecessary node creation by catching - * cases where old nodes can be reused because their next fields - * won't change. On average, only about one-sixth of them need - * cloning when a table doubles. The nodes they replace will be - * garbage collectable as soon as they are no longer referenced by - * any reader thread that may be in the midst of concurrently - * traversing table. Upon transfer, the old table bin contains - * only a special forwarding node (with hash field "MOVED") that - * contains the next table as its key. On encountering a - * forwarding node, access and update operations restart, using - * the new table. - * - * Each bin transfer requires its bin lock, which can stall - * waiting for locks while resizing. However, because other - * threads can join in and help resize rather than contend for - * locks, average aggregate waits become shorter as resizing - * progresses. The transfer operation must also ensure that all - * accessible bins in both the old and new table are usable by any - * traversal. This is arranged in part by proceeding from the - * last bin (table.length - 1) up towards the first. Upon seeing - * a forwarding node, traversals (see class Traverser) arrange to - * move to the new table without revisiting nodes. To ensure that - * no intervening nodes are skipped even when moved out of order, - * a stack (see class TableStack) is created on first encounter of - * a forwarding node during a traversal, to maintain its place if - * later processing the current table. The need for these - * save/restore mechanics is relatively rare, but when one - * forwarding node is encountered, typically many more will be. - * So Traversers use a simple caching scheme to avoid creating so - * many new TableStack nodes. (Thanks to Peter Levart for - * suggesting use of a stack here.) - * - * The traversal scheme also applies to partial traversals of - * ranges of bins (via an alternate Traverser constructor) - * to support partitioned aggregate operations. Also, read-only - * operations give up if ever forwarded to a null table, which - * provides support for shutdown-style clearing, which is also not - * currently implemented. - * - * Lazy table initialization minimizes footprint until first use, - * and also avoids resizings when the first operation is from a - * putAll, constructor with map argument, or deserialization. - * These cases attempt to override the initial capacity settings, - * but harmlessly fail to take effect in cases of races. - * - * The element count is maintained using a specialization of - * LongAdder. We need to incorporate a specialization rather than - * just use a LongAdder in order to access implicit - * contention-sensing that leads to creation of multiple - * CounterCells. The counter mechanics avoid contention on - * updates but can encounter cache thrashing if read too - * frequently during concurrent access. To avoid reading so often, - * resizing under contention is attempted only upon adding to a - * bin already holding two or more nodes. Under uniform hash - * distributions, the probability of this occurring at threshold - * is around 13%, meaning that only about 1 in 8 puts check - * threshold (and after resizing, many fewer do so). - * - * TreeBins use a special form of comparison for search and - * related operations (which is the main reason we cannot use - * existing collections such as TreeMaps). TreeBins contain - * Comparable elements, but may contain others, as well as - * elements that are Comparable but not necessarily Comparable for - * the same T, so we cannot invoke compareTo among them. To handle - * this, the tree is ordered primarily by hash value, then by - * Comparable.compareTo order if applicable. On lookup at a node, - * if elements are not comparable or compare as 0 then both left - * and right children may need to be searched in the case of tied - * hash values. (This corresponds to the full list search that - * would be necessary if all elements were non-Comparable and had - * tied hashes.) On insertion, to keep a total ordering (or as - * close as is required here) across rebalancings, we compare - * classes and identityHashCodes as tie-breakers. The red-black - * balancing code is updated from pre-jdk-collections - * (http://gee.cs.oswego.edu/dl/classes/collections/RBCell.java) - * based in turn on Cormen, Leiserson, and Rivest "Introduction to - * Algorithms" (CLR). - * - * TreeBins also require an additional locking mechanism. While - * list traversal is always possible by readers even during - * updates, tree traversal is not, mainly because of tree-rotations - * that may change the root node and/or its linkages. TreeBins - * include a simple read-write lock mechanism parasitic on the - * main bin-synchronization strategy: Structural adjustments - * associated with an insertion or removal are already bin-locked - * (and so cannot conflict with other writers) but must wait for - * ongoing readers to finish. Since there can be only one such - * waiter, we use a simple scheme using a single "waiter" field to - * block writers. However, readers need never block. If the root - * lock is held, they proceed along the slow traversal path (via - * next-pointers) until the lock becomes available or the list is - * exhausted, whichever comes first. These cases are not fast, but - * maximize aggregate expected throughput. - * - * Maintaining API and serialization compatibility with previous - * versions of this class introduces several oddities. Mainly: We - * leave untouched but unused constructor arguments referring to - * concurrencyLevel. We accept a loadFactor constructor argument, - * but apply it only to initial table capacity (which is the only - * time that we can guarantee to honor it.) We also declare an - * unused "Segment" class that is instantiated in minimal form - * only when serializing. - * - * Also, solely for compatibility with previous versions of this - * class, it extends AbstractMap, even though all of its methods - * are overridden, so it is just useless baggage. - * - * This file is organized to make things a little easier to follow - * while reading than they might otherwise: First the main static - * declarations and utilities, then fields, then main public - * methods (with a few factorings of multiple public methods into - * internal ones), then sizing methods, trees, traversers, and - * bulk operations. - */ - - /* ---------------- Constants -------------- */ - static final int HASH_BITS = 0x7fffffff; // usable bits of normal node hash - /** - * The largest possible (non-power of two) array size. - * Needed by toArray and related methods. - */ - static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; - /** - * The smallest table capacity for which bins may be treeified. - * (Otherwise the table is resized if too many nodes in a bin.) - * The value should be at least 4 * TREEIFY_THRESHOLD to avoid - * conflicts between resizing and treeification thresholds. - */ - static final int MIN_TREEIFY_CAPACITY = 64; - /* - * Encodings for Node hash fields. See above for explanation. - */ - static final int MOVED = -1; // hash for forwarding nodes - /** - * Number of CPUS, to place bounds on some sizings - */ - static final int NCPU = Runtime.getRuntime().availableProcessors(); - static final int RESERVED = -3; // hash for transient reservations - static final int TREEBIN = -2; // hash for roots of trees - /** - * The bin count threshold for using a tree rather than list for a - * bin. Bins are converted to trees when adding an element to a - * bin with at least this many nodes. The value must be greater - * than 2, and should be at least 8 to mesh with assumptions in - * tree removal about conversion back to plain bins upon - * shrinkage. - */ - static final int TREEIFY_THRESHOLD = 8; - /** - * The bin count threshold for untreeifying a (split) bin during a - * resize operation. Should be less than TREEIFY_THRESHOLD, and at - * most 6 to mesh with shrinkage detection under removal. - */ - static final int UNTREEIFY_THRESHOLD = 6; - /* ---------------- Fields -------------- */ - private static final long ABASE; - private static final int ASHIFT; - /* - * Volatile access methods are used for table elements as well as - * elements of in-progress next table while resizing. All uses of - * the tab arguments must be null checked by callers. All callers - * also paranoically precheck that tab's length is not zero (or an - * equivalent check), thus ensuring that any index argument taking - * the form of a hash value anded with (length - 1) is a valid - * index. Note that, to be correct wrt arbitrary concurrency - * errors by users, these checks must operate on local variables, - * which accounts for some odd-looking inline assignments below. - * Note that calls to setTabAt always occur within locked regions, - * and so in principle require only release ordering, not - * full volatile semantics, but are currently coded as volatile - * writes to be conservative. - */ - private static final long BASECOUNT; - private static final long CELLSBUSY; - private static final long CELLVALUE; - /** - * The default initial table capacity. Must be a power of 2 - * (i.e., at least 1) and at most MAXIMUM_CAPACITY. - */ - private static final int DEFAULT_CAPACITY = 16; - /** - * The load factor for this table. Overrides of this value in - * constructors affect only the initial table capacity. The - * actual floating point value isn't normally used -- it is - * simpler to use expressions such as {@code n - (n >>> 2)} for - * the associated resizing threshold. - */ - private static final float LOAD_FACTOR = 0.75f; - /** - * The largest possible table capacity. This value must be - * exactly 1<<30 to stay within Java array allocation and indexing - * bounds for power of two table sizes, and is further required - * because the top two bits of 32bit hash fields are used for - * control purposes. - */ - private static final int MAXIMUM_CAPACITY = 1 << 30; - /** - * Minimum number of rebinnings per transfer step. Ranges are - * subdivided to allow multiple resizer threads. This value - * serves as a lower bound to avoid resizers encountering - * excessive memory contention. The value should be at least - * DEFAULT_CAPACITY. - */ - private static final int MIN_TRANSFER_STRIDE = 16; - private static final long PROBE; - - /* ---------------- Nodes -------------- */ - /** - * The increment for generating probe values - */ - private static final int PROBE_INCREMENT = 0x9e3779b9; - - /* ---------------- Static utilities -------------- */ - /** - * The number of bits used for generation stamp in sizeCtl. - * Must be at least 6 for 32bit arrays. - */ - private static final int RESIZE_STAMP_BITS = 16; - /** - * The maximum number of threads that can help resize. - * Must fit in 32 - RESIZE_STAMP_BITS bits. - */ - private static final int MAX_RESIZERS = (1 << (32 - RESIZE_STAMP_BITS)) - 1; - /** - * The bit shift for recording size stamp in sizeCtl. - */ - private static final int RESIZE_STAMP_SHIFT = 32 - RESIZE_STAMP_BITS; - private static final long SEED; - - /* ---------------- Table element access -------------- */ - /** - * The increment of seeder per new instance - */ - private static final long SEEDER_INCREMENT = 0xbb67ae8584caa73bL; - private static final long SIZECTL; - private static final long TRANSFERINDEX; - /** - * Generates per-thread initialization/probe field - */ - private static final AtomicInteger probeGenerator = new AtomicInteger(); - /** - * The next seed for default constructors. - */ - private static final AtomicLong seeder = new AtomicLong(initialSeed()); - /** - * For serialization compatibility. - */ - private static final ObjectStreamField[] serialPersistentFields = { - new ObjectStreamField("segments", Segment[].class), - new ObjectStreamField("segmentMask", Integer.TYPE), - new ObjectStreamField("segmentShift", Integer.TYPE) - }; - private static final long serialVersionUID = 7249069246763182397L; - private final ThreadLocal> tlTraverser = ThreadLocal.withInitial(Traverser::new); - /** - * The array of bins. Lazily initialized upon first insertion. - * Size is always a power of two. Accessed directly by iterators. - */ - transient volatile Node[] table; - /** - * Base counter value, used mainly when there is no contention, - * but also as a fallback during table initialization - * races. Updated via CAS. - */ - private transient volatile long baseCount; - /** - * Spinlock (locked via CAS) used when resizing and/or creating CounterCells. - */ - private transient volatile int cellsBusy; - /** - * Table of counter cells. When non-null, size is a power of 2. - */ - private transient volatile CounterCell[] counterCells; - // Original (since JDK1.2) Map methods - private transient EntrySetView entrySet; - /* ---------------- Public operations -------------- */ - // views - private transient KeySetView keySet; - /** - * The next table to use; non-null only while resizing. - */ - private transient volatile Node[] nextTable; - /** - * Table initialization and resizing control. When negative, the - * table is being initialized or resized: -1 for initialization, - * else -(1 + the number of active resizing threads). Otherwise, - * when table is null, holds the initial table size to use upon - * creation, or 0 for default. After initialization, holds the - * next element count value upon which to resize the table. - */ - private transient volatile int sizeCtl; - /** - * The next table index (plus one) to split while resizing. - */ - private transient volatile int transferIndex; - private transient ValuesView values; - - /** - * Creates a new, empty map with an initial table size based on - * the given number of elements ({@code initialCapacity}), table - * density ({@code loadFactor}), and number of concurrently - * updating threads ({@code concurrencyLevel}). - * - * @param initialCapacity the initial capacity. The implementation - * performs internal sizing to accommodate this many elements, - * given the specified load factor. - * @param loadFactor the load factor (table density) for - * establishing the initial table size - * @throws IllegalArgumentException if the initial capacity is - * negative or the load factor or concurrencyLevel are - * nonpositive - */ - public ConcurrentIntHashMap(int initialCapacity, float loadFactor) { - if (!(loadFactor > 0.0f) || initialCapacity < 0) - throw new IllegalArgumentException(); - if (initialCapacity < 1) // Use at least as many bins - initialCapacity = 1; // as estimated threads - long size = (long) (1.0 + (long) initialCapacity / loadFactor); - this.sizeCtl = (size >= (long) MAXIMUM_CAPACITY) ? - MAXIMUM_CAPACITY : tableSizeFor((int) size); - } - - /** - * Creates a new map with the same mappings as the given map. - * - * @param m the map - */ - public ConcurrentIntHashMap(ConcurrentIntHashMap m) { - this.sizeCtl = DEFAULT_CAPACITY; - putAll(m); - } - - /** - * Creates a new, empty map with the default initial table size (16). - */ - public ConcurrentIntHashMap() { - } - - /** - * Creates a new, empty map with an initial table size - * accommodating the specified number of elements without the need - * to dynamically resize. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - */ - public ConcurrentIntHashMap(int initialCapacity) { - if (initialCapacity < 0) - throw new IllegalArgumentException(); - this.sizeCtl = ((initialCapacity >= (MAXIMUM_CAPACITY >>> 1)) ? - MAXIMUM_CAPACITY : - tableSizeFor(initialCapacity + (initialCapacity >>> 1) + 1)); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentLongHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @return the new set - * @since 1.8 - */ - public static KeySetView newKeySet() { - return new KeySetView<>(new ConcurrentIntHashMap<>(), Boolean.TRUE); - } - - /** - * Creates a new {@link Set} backed by a ConcurrentLongHashMap - * from the given type to {@code Boolean.TRUE}. - * - * @param initialCapacity The implementation performs internal - * sizing to accommodate this many elements. - * @return the new set - * @throws IllegalArgumentException if the initial capacity of - * elements is negative - * @since 1.8 - */ - public static KeySetView newKeySet(int initialCapacity) { - return new KeySetView<>(new ConcurrentIntHashMap<>(initialCapacity), Boolean.TRUE); - } - - /** - * Removes all of the mappings from this map. - */ - public void clear() { - long delta = 0L; // negative number of deletions - int i = 0; - Node[] tab = table; - while (tab != null && i < tab.length) { - int fh; - Node f = tabAt(tab, i); - if (f == null) - ++i; - else if ((fh = f.hash) == MOVED) { - tab = helpTransfer(tab, f); - i = 0; // restart - } else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node p = (fh >= 0 ? f : - (f instanceof TreeBin) ? - ((TreeBin) f).first : null); - while (p != null) { - --delta; - p = p.next; - } - setTabAt(tab, i++, null); - } - } - } - } - if (delta != 0L) - addCount(delta, -1); - } - - /** - * Attempts to compute a mapping for the specified key and its - * current mapped value (or {@code null} if there is no current - * mapping). The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this Map. - * - * @param key key with which the specified value is to be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified remappingFunction is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V compute(int key, BiIntFunction remappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = remappingFunction.apply(key, null)) != null) { - delta = 1; - node = new Node<>(h, key, val, null); - } - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - if (e.hash == h && (e.key == key)) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) { - val = remappingFunction.apply(key, null); - if (val != null) { - delta = 1; - pred.next = new Node<>(h, key, val, null); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 1; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null) - p = r.findTreeNode(h, key); - else - p = null; - V pv = (p == null) ? null : p.val; - val = remappingFunction.apply(key, pv); - if (val != null) { - if (p != null) - p.val = val; - else { - delta = 1; - t.putTreeVal(h, key, val); - } - } else if (p != null) { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - break; - } - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param token token to pass to the mapping function - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified mappingFunction is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(int key, Object token, BiIntFunction mappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key, token)) != null) - node = new Node<>(h, key, val, null); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - if (e.hash == h && e.key == key) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - pred.next = new Node<>(h, key, val, null); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key, token)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the specified key is not already associated with a value, - * attempts to compute its value using the given mapping function - * and enters it into this map unless {@code null}. The entire - * method invocation is performed atomically, so the function is - * applied at most once per key. Some attempted update operations - * on this map by other threads may be blocked while computation - * is in progress, so the computation should be short and simple, - * and must not attempt to update any other mappings of this map. - * - * @param key key with which the specified value is to be associated - * @param mappingFunction the function to compute a value - * @return the current (existing or computed) value associated with - * the specified key, or null if the computed value is null - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified key or mappingFunction - * is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the mappingFunction does so, - * in which case the mapping is left unestablished - */ - public V computeIfAbsent(int key, IntFunction mappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (mappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) { - Node r = new ReservationNode<>(); - synchronized (r) { - if (casTabAt(tab, i, r)) { - binCount = 1; - Node node = null; - try { - if ((val = mappingFunction.apply(key)) != null) - node = new Node<>(h, key, val, null); - } finally { - setTabAt(tab, i, node); - } - } - } - if (binCount != 0) - break; - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - boolean added = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - if (e.hash == h && e.key == key) { - val = e.val; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if ((val = mappingFunction.apply(key)) != null) { - added = true; - pred.next = new Node<>(h, key, val, null); - } - break; - } - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key)) != null) - val = p.val; - else if ((val = mappingFunction.apply(key)) != null) { - added = true; - t.putTreeVal(h, key, val); - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (!added) - return val; - break; - } - } - } - if (val != null) - addCount(1L, binCount); - return val; - } - - /** - * If the value for the specified key is present, attempts to - * compute a new mapping given the key and its current mapped - * value. The entire method invocation is performed atomically. - * Some attempted update operations on this map by other threads - * may be blocked while computation is in progress, so the - * computation should be short and simple, and must not attempt to - * update any other mappings of this map. - * - * @param key key with which a value may be associated - * @param remappingFunction the function to compute a value - * @return the new value associated with the specified key, or null if none - * @throws IllegalArgumentException if the specified key is negative - * @throws NullPointerException if the specified remappingFunction is null - * @throws IllegalStateException if the computation detectably - * attempts a recursive update to this map that would - * otherwise never complete - * @throws RuntimeException or Error if the remappingFunction does so, - * in which case the mapping is unchanged - */ - public V computeIfPresent(int key, BiIntFunction remappingFunction) { - if (key < 0) - throw new IllegalArgumentException(); - if (remappingFunction == null) - throw new NullPointerException(); - int h = spread(keyHashCode(key)); - V val = null; - int delta = 0; - int binCount = 0; - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & h)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f, pred = null; ; ++binCount) { - if (e.hash == h && e.key == key) { - val = remappingFunction.apply(key, e.val); - if (val != null) - e.val = val; - else { - delta = -1; - Node en = e.next; - if (pred != null) - pred.next = en; - else - setTabAt(tab, i, en); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - binCount = 2; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(h, key)) != null) { - val = remappingFunction.apply(key, p.val); - if (val != null) - p.val = val; - else { - delta = -1; - if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (binCount != 0) - break; - } - } - if (delta != 0) - addCount(delta, binCount); - return val; - } - - /** - * Tests if the specified object is a key in this table. - * - * @param key possible key - * @return {@code true} if and only if the specified object - * is a key in this table, as determined by the - * {@code equals} method; {@code false} otherwise - * @throws NullPointerException if the specified key is null - */ - public boolean containsKey(int key) { - return get(key) != null; - } - - /** - * Returns {@code true} if this map maps one or more keys to the - * specified value. Note: This method may require a full traversal - * of the map, and is much slower than method {@code containsKey}. - * - * @param value value whose presence in this map is to be tested - * @return {@code true} if this map maps one or more keys to the - * specified value - * @throws NullPointerException if the specified value is null - */ - public boolean containsValue(V value) { - if (value == null) - throw new NullPointerException(); - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - V v; - if ((v = p.val) == value || (value.equals(v))) - return true; - } - } - return false; - } - - /** - * Returns a {@link Set} view of the mappings contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from the map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the set view - */ - @NotNull - public Set> entrySet() { - EntrySetView es; - return (es = entrySet) != null ? es : (entrySet = new EntrySetView<>(this)); - } - - /** - * Compares the specified object with this map for equality. - * Returns {@code true} if the given object is a map with the same - * mappings as this map. This operation may return misleading - * results if either map is concurrently modified during execution - * of this method. - * - * @param o object to be compared for equality with this map - * @return {@code true} if the specified object is equal to this map - */ - public boolean equals(Object o) { - if (o != this) { - if (!(o instanceof ConcurrentIntHashMap)) - return false; - ConcurrentIntHashMap m = (ConcurrentIntHashMap) o; - Traverser it = getTraverser(table); - for (Node p; (p = it.advance()) != null; ) { - V val = p.val; - Object v = m.get(p.key); - if (v == null || (v != val && !v.equals(val))) - return false; - } - for (IntEntry e : m.entrySet()) { - int mk; - Object mv, v; - if ((mk = e.getKey()) == EMPTY_KEY || - (mv = e.getValue()) == null || - (v = get(mk)) == null || - (mv != v && !mv.equals(v))) - return false; - } - } - return true; - } - - /** - * Returns the value to which the specified key is mapped, - * or {@code null} if this map contains no mapping for the key. - *

    More formally, if this map contains a mapping from a key - * {@code k} to a value {@code v} such that {@code key.equals(k)}, - * then this method returns {@code v}; otherwise it returns - * {@code null}. (There can be at most one such mapping.) - * - * @param key map key value - * @return value to which specified key is mapped - * @throws NullPointerException if the specified key is null - */ - public V get(int key) { - Node[] tab; - Node e, p; - int n, eh; - int h = spread(keyHashCode(key)); - if ((tab = table) != null && (n = tab.length) > 0 && - (e = tabAt(tab, (n - 1) & h)) != null) { - if ((eh = e.hash) == h) { - if (e.key == key) - return e.val; - } else if (eh < 0) - return (p = e.find(h, key)) != null ? p.val : null; - while ((e = e.next) != null) { - if (e.hash == h && e.key == key) - return e.val; - } - } - return null; - } - - /** - * Returns the value to which the specified key is mapped, or the - * given default value if this map contains no mapping for the - * key. - * - * @param key the key whose associated value is to be returned - * @param defaultValue the value to return if this map contains - * no mapping for the given key - * @return the mapping for the key, if present; else the default value - * @throws NullPointerException if the specified key is null - */ - public V getOrDefault(int key, V defaultValue) { - V v; - return (v = get(key)) == null ? defaultValue : v; - } - - /** - * Returns the hash code value for this {@link Map}, i.e., - * the sum of, for each key-value pair in the map, - * {@code key.hashCode() ^ value.hashCode()}. - * - * @return the hash code value for this map - */ - public int hashCode() { - int h = 0; - Node[] t = table; - if (t != null) { - Traverser it = getTraverser(t); - for (Node p; (p = it.advance()) != null; ) - h += keyHashCode(p.key) ^ p.val.hashCode(); - } - return h; - } - - // ConcurrentMap methods - - /** - * {@inheritDoc} - */ - public boolean isEmpty() { - return sumCount() <= 0L; // ignore transient negative values - } - - /** - * Returns a {@link Set} view of the keys in this map, using the - * given common mapped value for any additions (i.e., {@link - * Collection#add} and {@link Collection#addAll(Collection)}). - * This is of course only appropriate if it is acceptable to use - * the same value for all additions from this view. - * - * @param mappedValue the mapped value to use for any additions - * @return the set view - * @throws NullPointerException if the mappedValue is null - */ - public KeySetView keySet(V mappedValue) { - if (mappedValue == null) - throw new NullPointerException(); - return new KeySetView<>(this, mappedValue); - } - - /** - * Returns a {@link Set} view of the keys contained in this map. - * The set is backed by the map, so changes to the map are - * reflected in the set, and vice-versa. The set supports element - * removal, which removes the corresponding mapping from this map, - * via the {@code Iterator.remove}, {@code Set.remove}, - * {@code removeAll}, {@code retainAll}, and {@code clear} - * operations. It does not support the {@code add} or - * {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - *

    - * - * @return the set view - */ - @NotNull - public KeySetView keySet() { - KeySetView ks; - return (ks = keySet) != null ? ks : (keySet = new KeySetView<>(this, null)); - } - - /** - * Returns the number of mappings. This method should be used - * instead of {@link #size} because a ConcurrentLongHashMap may - * contain more mappings than can be represented as an int. The - * value returned is an estimate; the actual count may differ if - * there are concurrent insertions or removals. - * - * @return the number of mappings - * @since 1.8 - */ - public long mappingCount() { - return Math.max(sumCount(), 0L); // ignore transient negative values - } - - // Overrides of JDK8+ Map extension method defaults - - /** - * Maps the specified key to the specified value in this table. - * Neither the key nor the value can be null. - *

    The value can be retrieved by calling the {@code get} method - * with a key that is equal to the original key. - * - * @param key key with which the specified value is to be associated - * @param value value to be associated with the specified key - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key or value is null - */ - public V put(int key, V value) { - return putVal(key, value, false); - } - - /** - * Copies all of the mappings from the specified map to this one. - * These mappings replace any mappings that this map had for any of the - * keys currently in the specified map. - * - * @param m mappings to be stored in this map - */ - public void putAll(@NotNull ConcurrentIntHashMap m) { - tryPresize(m.size()); - for (IntEntry e : m.entrySet()) - putVal(e.getKey(), e.getValue(), false); - } - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V putIfAbsent(int key, V value) { - return putVal(key, value, true); - } - - public boolean remove(int key, V value) { - return value != null && replaceNode(key, null, value) != null; - } - - /** - * Removes the key (and its corresponding value) from this map. - * This method does nothing if the key is not in the map. - * - * @param key the key that needs to be removed - * @return the previous value associated with {@code key}, or - * {@code null} if there was no mapping for {@code key} - * @throws NullPointerException if the specified key is null - */ - public V remove(int key) { - return replaceNode(key, null, null); - } - - // Hashtable legacy methods - - public boolean replace(int key, @NotNull V oldValue, @NotNull V newValue) { - return replaceNode(key, newValue, oldValue) != null; - } - - // ConcurrentLongHashMap-only methods - - /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, - * or {@code null} if there was no mapping for the key - * @throws NullPointerException if the specified key or value is null - */ - public V replace(int key, @NotNull V value) { - return replaceNode(key, value, null); - } - - /** - * {@inheritDoc} - */ - public int size() { - long n = sumCount(); - return ((n < 0L) ? 0 : - (n > (long) Integer.MAX_VALUE) ? Integer.MAX_VALUE : - (int) n); - } - - /** - * Returns a string representation of this map. The string - * representation consists of a list of key-value mappings (in no - * particular order) enclosed in braces ("{@code {}}"). Adjacent - * mappings are separated by the characters {@code ", "} (comma - * and space). Each key-value mapping is rendered as the key - * followed by an equals sign ("{@code =}") followed by the - * associated value. - * - * @return a string representation of this map - */ - public String toString() { - Traverser it = getTraverser(table); - StringBuilder sb = new StringBuilder(); - sb.append('{'); - Node p; - if ((p = it.advance()) != null) { - for (; ; ) { - int k = p.key; - V v = p.val; - sb.append(k); - sb.append('='); - sb.append(v == this ? "(this Map)" : v); - if ((p = it.advance()) == null) - break; - sb.append(',').append(' '); - } - } - return sb.append('}').toString(); - } - - /** - * Returns a {@link Collection} view of the values contained in this map. - * The collection is backed by the map, so changes to the map are - * reflected in the collection, and vice-versa. The collection - * supports element removal, which removes the corresponding - * mapping from this map, via the {@code Iterator.remove}, - * {@code Collection.remove}, {@code removeAll}, - * {@code retainAll}, and {@code clear} operations. It does not - * support the {@code add} or {@code addAll} operations. - *

    The view's iterators and spliterators are - * weakly consistent. - * - * @return the collection view - */ - @NotNull - public Collection values() { - ValuesView vs; - return (vs = values) != null ? vs : (values = new ValuesView<>(this)); - } - - /* ---------------- Special Nodes -------------- */ - - private static long initialSeed() { - String pp = System.getProperty("java.util.secureRandomSeed"); - - if (pp != null && pp.equalsIgnoreCase("true")) { - byte[] seedBytes = java.security.SecureRandom.getSeed(8); - long s = (long) (seedBytes[0]) & 0xffL; - for (int i = 1; i < 8; ++i) - s = (s << 8) | ((long) (seedBytes[i]) & 0xffL); - return s; - } - return (mix64(System.currentTimeMillis()) ^ - mix64(System.nanoTime())); - } - - /* ---------------- Table Initialization and Resizing -------------- */ - - private static int keyHashCode(int key) { - return key; - } - - private static long mix64(long z) { - z = (z ^ (z >>> 33)) * 0xff51afd7ed558ccdL; - z = (z ^ (z >>> 33)) * 0xc4ceb9fe1a85ec53L; - return z ^ (z >>> 33); - } - - /** - * Returns a power of two table size for the given desired capacity. - * See Hackers Delight, sec 3.2 - */ - private static int tableSizeFor(int c) { - int n = c - 1; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - return (n < 0) ? 1 : (n >= MAXIMUM_CAPACITY) ? MAXIMUM_CAPACITY : n + 1; - } - - /** - * Adds to count, and if table is too small and not already - * resizing, initiates transfer. If already resizing, helps - * perform transfer if work is available. Rechecks occupancy - * after a transfer to see if another resize is already needed - * because resizings are lagging additions. - * - * @param x the count to add - * @param check if <0, don't check resize, if <= 1 only check if uncontended - */ - private void addCount(long x, int check) { - CounterCell[] as; - long b, s; - if ((as = counterCells) != null || !Unsafe.cas(this, BASECOUNT, b = baseCount, s = b + x)) { - CounterCell a; - long v; - int m; - boolean uncontended = true; - if (as == null || (m = as.length - 1) < 0 || - (a = as[getProbe() & m]) == null || - !(uncontended = Unsafe.cas(a, CELLVALUE, v = a.value, v + x))) { - fullAddCount(x, uncontended); - return; - } - if (check <= 1) - return; - s = sumCount(); - } - if (check >= 0) { - Node[] tab, nt; - int n, sc; - while (s >= (long) (sc = sizeCtl) && (tab = table) != null && - (n = tab.length) < MAXIMUM_CAPACITY) { - int rs = resizeStamp(n); - if (sc < 0) { - if (sc >>> RESIZE_STAMP_SHIFT != rs || sc == rs + MAX_RESIZERS || (nt = nextTable) == null || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - s = sumCount(); - } - } - } - - // See LongAdder version for explanation - private void fullAddCount(long x, boolean wasUncontended) { - int h; - if ((h = getProbe()) == 0) { - localInit(); // force initialization - h = getProbe(); - wasUncontended = true; - } - boolean collide = false; // True if last slot nonempty - for (; ; ) { - CounterCell[] as; - CounterCell a; - int n; - long v; - if ((as = counterCells) != null && (n = as.length) > 0) { - if ((a = as[(n - 1) & h]) == null) { - if (cellsBusy == 0) { // Try to attach new Cell - CounterCell r = new CounterCell(x); // Optimistic create - if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean created = false; - try { // Recheck under lock - CounterCell[] rs; - int m, j; - if ((rs = counterCells) != null && - (m = rs.length) > 0 && - rs[j = (m - 1) & h] == null) { - rs[j] = r; - created = true; - } - } finally { - cellsBusy = 0; - } - if (created) - break; - continue; // Slot is now non-empty - } - } - collide = false; - } else if (!wasUncontended) // CAS already known to fail - wasUncontended = true; // Continue after rehash - else if (Unsafe.cas(a, CELLVALUE, v = a.value, v + x)) - break; - else if (counterCells != as || n >= NCPU) - collide = false; // At max size or stale - else if (!collide) - collide = true; - else if (cellsBusy == 0 && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - try { - if (counterCells == as) {// Expand table unless stale - CounterCell[] rs = new CounterCell[n << 1]; - System.arraycopy(as, 0, rs, 0, n); - counterCells = rs; - } - } finally { - cellsBusy = 0; - } - collide = false; - continue; // Retry with expanded table - } - h = advanceProbe(h); - } else if (cellsBusy == 0 && counterCells == as && - Unsafe.getUnsafe().compareAndSwapInt(this, CELLSBUSY, 0, 1)) { - boolean init = false; - try { // Initialize table - if (counterCells == as) { - CounterCell[] rs = new CounterCell[2]; - rs[h & 1] = new CounterCell(x); - counterCells = rs; - init = true; - } - } finally { - cellsBusy = 0; - } - if (init) - break; - } else if (Unsafe.cas(this, BASECOUNT, v = baseCount, v + x)) - break; // Fall back on using base - } - } - - private Traverser getTraverser(Node[] tab) { - Traverser traverser = tlTraverser.get(); - int len = tab == null ? 0 : tab.length; - traverser.of(tab, len, len); - return traverser; - } - - /** - * Initializes table, using the size recorded in sizeCtl. - */ - private Node[] initTable() { - Node[] tab; - int sc; - while ((tab = table) == null || tab.length == 0) { - if ((sc = sizeCtl) < 0) - Os.pause(); // lost initialization race; just spin - else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if ((tab = table) == null || tab.length == 0) { - int n = (sc > 0) ? sc : DEFAULT_CAPACITY; - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = tab = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - break; - } - } - return tab; - } - - /** - * Moves and/or copies the nodes in each bin to new table. See - * above for explanation. - */ - private void transfer(Node[] tab, Node[] nextTab) { - int n = tab.length, stride; - if ((stride = (NCPU > 1) ? (n >>> 3) / NCPU : n) < MIN_TRANSFER_STRIDE) - stride = MIN_TRANSFER_STRIDE; // subdivide range - if (nextTab == null) { // initiating - try { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n << 1]; - nextTab = nt; - } catch (Throwable ex) { // try to cope with OOME - sizeCtl = Integer.MAX_VALUE; - return; - } - nextTable = nextTab; - transferIndex = n; - } - int nextn = nextTab.length; - ForwardingNode fwd = new ForwardingNode<>(nextTab); - boolean advance = true; - boolean finishing = false; // to ensure sweep before committing nextTab - for (int i = 0, bound = 0; ; ) { - Node f; - int fh; - while (advance) { - int nextIndex, nextBound; - if (--i >= bound || finishing) - advance = false; - else if ((nextIndex = transferIndex) <= 0) { - i = -1; - advance = false; - } else if (Unsafe.getUnsafe().compareAndSwapInt - (this, TRANSFERINDEX, nextIndex, - nextBound = (nextIndex > stride ? - nextIndex - stride : 0))) { - bound = nextBound; - i = nextIndex - 1; - advance = false; - } - } - if (i < 0 || i >= n || i + n >= nextn) { - int sc; - if (finishing) { - nextTable = null; - table = nextTab; - sizeCtl = (n << 1) - (n >>> 1); - return; - } - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc = sizeCtl, sc - 1)) { - if ((sc - 2) != resizeStamp(n) << RESIZE_STAMP_SHIFT) - return; - finishing = advance = true; - i = n; // recheck before commit - } - } else if ((f = tabAt(tab, i)) == null) - advance = casTabAt(tab, i, fwd); - else if ((fh = f.hash) == MOVED) - advance = true; // already processed - else { - synchronized (f) { - if (tabAt(tab, i) == f) { - Node ln, hn; - if (fh >= 0) { - int runBit = fh & n; - Node lastRun = f; - for (Node p = f.next; p != null; p = p.next) { - int b = p.hash & n; - if (b != runBit) { - runBit = b; - lastRun = p; - } - } - if (runBit == 0) { - ln = lastRun; - hn = null; - } else { - hn = lastRun; - ln = null; - } - for (Node p = f; p != lastRun; p = p.next) { - int ph = p.hash; - int pk = p.key; - V pv = p.val; - if ((ph & n) == 0) - ln = new Node<>(ph, pk, pv, ln); - else - hn = new Node<>(ph, pk, pv, hn); - } - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } else if (f instanceof TreeBin) { - TreeBin t = (TreeBin) f; - TreeNode lo = null, loTail = null; - TreeNode hi = null, hiTail = null; - int lc = 0, hc = 0; - for (Node e = t.first; e != null; e = e.next) { - int h = e.hash; - TreeNode p = new TreeNode<>(h, e.key, e.val, null, null); - if ((h & n) == 0) { - if ((p.prev = loTail) == null) - lo = p; - else - loTail.next = p; - loTail = p; - ++lc; - } else { - if ((p.prev = hiTail) == null) - hi = p; - else - hiTail.next = p; - hiTail = p; - ++hc; - } - } - ln = (lc <= UNTREEIFY_THRESHOLD) ? untreeify(lo) : - (hc != 0) ? new TreeBin<>(lo) : t; - hn = (hc <= UNTREEIFY_THRESHOLD) ? untreeify(hi) : - (lc != 0) ? new TreeBin<>(hi) : t; - setTabAt(nextTab, i, ln); - setTabAt(nextTab, i + n, hn); - setTabAt(tab, i, fwd); - advance = true; - } - } - } - } - } - } - /* ---------------- Counter support -------------- */ - - /** - * Replaces all linked nodes in bin at given index unless table is - * too small, in which case resizes instead. - */ - private void treeifyBin(Node[] tab, int index) { - Node b; - int n; - if (tab != null) { - if ((n = tab.length) < MIN_TREEIFY_CAPACITY) - tryPresize(n << 1); - else if ((b = tabAt(tab, index)) != null && b.hash >= 0) { - synchronized (b) { - if (tabAt(tab, index) == b) { - TreeNode hd = null, tl = null; - for (Node e = b; e != null; e = e.next) { - TreeNode p = - new TreeNode<>(e.hash, e.key, e.val, null, null); - if ((p.prev = tl) == null) - hd = p; - else - tl.next = p; - tl = p; - } - setTabAt(tab, index, new TreeBin<>(hd)); - } - } - } - } - } - - /** - * Tries to presize table to accommodate the given number of elements. - * - * @param size number of elements (doesn't need to be perfectly accurate) - */ - private void tryPresize(int size) { - int c = (size >= (MAXIMUM_CAPACITY >>> 1)) ? MAXIMUM_CAPACITY : - tableSizeFor(size + (size >>> 1) + 1); - int sc; - while ((sc = sizeCtl) >= 0) { - Node[] tab = table; - int n; - if (tab == null || (n = tab.length) == 0) { - n = Math.max(sc, c); - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, -1)) { - try { - if (table == tab) { - @SuppressWarnings("unchecked") - Node[] nt = (Node[]) new Node[n]; - table = nt; - sc = n - (n >>> 2); - } - } finally { - sizeCtl = sc; - } - } - } else if (c <= sc || n >= MAXIMUM_CAPACITY) - break; - else if (tab == table) { - int rs = resizeStamp(n); - if (sc < 0) { - Node[] nt; - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || (nt = nextTable) == null || - transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) - transfer(tab, nt); - } else if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, - (rs << RESIZE_STAMP_SHIFT) + 2)) - transfer(tab, null); - } - } - } - - static int advanceProbe(int probe) { - probe ^= probe << 13; // xorshift - probe ^= probe >>> 17; - probe ^= probe << 5; - Unsafe.getUnsafe().putInt(Thread.currentThread(), PROBE, probe); - return probe; - } - - /* ---------------- Conversion from/to TreeBins -------------- */ - - static boolean casTabAt(Node[] tab, int i, - Node v) { - return Unsafe.getUnsafe().compareAndSwapObject(tab, ((long) i << ASHIFT) + ABASE, null, v); - } - - /** - * Returns x's Class if it is of the form "class C implements - * Comparable", else null. - */ - static Class comparableClassFor(Object x) { - if (x instanceof Comparable) { - Class c; - Type[] ts, as; - Type t; - ParameterizedType p; - if ((c = x.getClass()) == String.class) // bypass checks - return c; - if ((ts = c.getGenericInterfaces()) != null) { - for (int i = 0; i < ts.length; ++i) { - if (((t = ts[i]) instanceof ParameterizedType) && - ((p = (ParameterizedType) t).getRawType() == - Comparable.class) && - (as = p.getActualTypeArguments()) != null && - as.length == 1 && as[0] == c) // type arg is c - return c; - } - } - } - return null; - } - - /* ---------------- TreeNodes -------------- */ - - /** - * Returns k.compareTo(x) if x matches kc (k's screened comparable - * class), else 0. - */ - static int compareComparables(int k, long x) { - return Long.compare(k, x); - } - - /* ---------------- TreeBins -------------- */ - - static int getProbe() { - return Unsafe.getUnsafe().getInt(Thread.currentThread(), PROBE); - } - - /* ----------------Table Traversal -------------- */ - - /** - * Initialize Thread fields for the current thread. Called only - * when Thread.threadLocalRandomProbe is zero, indicating that a - * thread local seed value needs to be generated. Note that even - * though the initialization is purely thread-local, we need to - * rely on (static) atomic generators to initialize the values. - */ - static void localInit() { - int p = probeGenerator.addAndGet(PROBE_INCREMENT); - int probe = (p == 0) ? 1 : p; // skip 0 - long seed = mix64(seeder.getAndAdd(SEEDER_INCREMENT)); - Thread t = Thread.currentThread(); - Unsafe.getUnsafe().putLong(t, SEED, seed); - Unsafe.getUnsafe().putInt(t, PROBE, probe); - } - - /** - * Returns the stamp bits for resizing a table of size n. - * Must be negative when shifted left by RESIZE_STAMP_SHIFT. - */ - static int resizeStamp(int n) { - return Integer.numberOfLeadingZeros(n) | (1 << (RESIZE_STAMP_BITS - 1)); - } - - static void setTabAt(Node[] tab, int i, Node v) { - Unsafe.getUnsafe().putObjectVolatile(tab, ((long) i << ASHIFT) + ABASE, v); - } - - /** - * Spreads (XORs) higher bits of hash to lower and also forces top - * bit to 0. Because the table uses power-of-two masking, sets of - * hashes that vary only in bits above the current mask will - * always collide. (Among known examples are sets of Float keys - * holding consecutive whole numbers in small tables.) So we - * apply a transform that spreads the impact of higher bits - * downward. There is a tradeoff between speed, utility, and - * quality of bit-spreading. Because many common sets of hashes - * are already reasonably distributed (so don't benefit from - * spreading), and because we use trees to handle large sets of - * collisions in bins, we just XOR some shifted bits in the - * cheapest possible way to reduce systematic lossage, as well as - * to incorporate impact of the highest bits that would otherwise - * never be used in index calculations because of table bounds. - */ - static int spread(int h) { - return (h ^ (h >>> 16)) & HASH_BITS; - } - - @SuppressWarnings("unchecked") - static Node tabAt(Node[] tab, int i) { - return (Node) Unsafe.getUnsafe().getObjectVolatile(tab, ((long) i << ASHIFT) + ABASE); - } - - /** - * Returns a list on non-TreeNodes replacing those in given list. - */ - static Node untreeify(Node b) { - Node hd = null, tl = null; - for (Node q = b; q != null; q = q.next) { - Node p = new Node<>(q.hash, q.key, q.val, null); - if (tl == null) - hd = p; - else - tl.next = p; - tl = p; - } - return hd; - } - - /** - * Helps transfer if a resize is in progress. - */ - final Node[] helpTransfer(Node[] tab, Node f) { - Node[] nextTab; - int sc; - if (tab != null && (f instanceof ForwardingNode) && - (nextTab = ((ForwardingNode) f).nextTable) != null) { - int rs = resizeStamp(tab.length); - while (nextTab == nextTable && table == tab && - (sc = sizeCtl) < 0) { - if ((sc >>> RESIZE_STAMP_SHIFT) != rs || sc == rs + 1 || - sc == rs + MAX_RESIZERS || transferIndex <= 0) - break; - if (Unsafe.getUnsafe().compareAndSwapInt(this, SIZECTL, sc, sc + 1)) { - transfer(tab, nextTab); - break; - } - } - return nextTab; - } - return table; - } - - /* ----------------Views -------------- */ - - /** - * Implementation for put and putIfAbsent - */ - final V putVal(int key, V value, boolean onlyIfAbsent) { - if (key < 0) throw new IllegalArgumentException(); - if (value == null) throw new NullPointerException(); - int hash = spread(keyHashCode(key)); - int binCount = 0; - Node _new = null; - - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0) - tab = initTable(); - else if ((f = tabAt(tab, i = (n - 1) & hash)) == null) { - if (_new == null) { - _new = new Node<>(hash, key, value, null); - } - if (casTabAt(tab, i, _new)) { - break; // no lock when adding to empty bin - } - } else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - binCount = 1; - for (Node e = f; ; ++binCount) { - if (e.hash == hash && e.key == key) { - oldVal = e.val; - if (!onlyIfAbsent) - e.val = value; - break; - } - Node pred = e; - if ((e = e.next) == null) { - if (_new == null) { - pred.next = new Node<>(hash, key, value, null); - } else { - pred.next = _new; - } - break; - } - } - } else if (f instanceof TreeBin) { - Node p; - binCount = 2; - if ((p = ((TreeBin) f).putTreeVal(hash, key, value)) != null) { - oldVal = p.val; - if (!onlyIfAbsent) - p.val = value; - } - } - } - } - if (binCount != 0) { - if (binCount >= TREEIFY_THRESHOLD) - treeifyBin(tab, i); - if (oldVal != null) - return oldVal; - break; - } - } - } - addCount(1L, binCount); - return null; - } - - /** - * Implementation for the four public remove/replace methods: - * Replaces node value with v, conditional upon match of cv if - * non-null. If resulting value is null, delete. - */ - final V replaceNode(int key, V value, V cv) { - int hash = spread(keyHashCode(key)); - for (Node[] tab = table; ; ) { - Node f; - int n, i, fh; - if (tab == null || (n = tab.length) == 0 || - (f = tabAt(tab, i = (n - 1) & hash)) == null) - break; - else if ((fh = f.hash) == MOVED) - tab = helpTransfer(tab, f); - else { - V oldVal = null; - boolean validated = false; - synchronized (f) { - if (tabAt(tab, i) == f) { - if (fh >= 0) { - validated = true; - for (Node e = f, pred = null; ; ) { - if (e.hash == hash && e.key == key) { - V ev = e.val; - if (cv == null || cv == ev || (cv.equals(ev))) { - oldVal = ev; - if (value != null) - e.val = value; - else if (pred != null) - pred.next = e.next; - else - setTabAt(tab, i, e.next); - } - break; - } - pred = e; - if ((e = e.next) == null) - break; - } - } else if (f instanceof TreeBin) { - validated = true; - TreeBin t = (TreeBin) f; - TreeNode r, p; - if ((r = t.root) != null && - (p = r.findTreeNode(hash, key)) != null) { - V pv = p.val; - if (cv == null || cv == pv || cv.equals(pv)) { - oldVal = pv; - if (value != null) - p.val = value; - else if (t.removeTreeNode(p)) - setTabAt(tab, i, untreeify(t.first)); - } - } - } - } - } - if (validated) { - if (oldVal != null) { - if (value == null) - addCount(-1L, -1); - return oldVal; - } - break; - } - } - } - return null; - } - - final long sumCount() { - CounterCell[] as = counterCells; - CounterCell a; - long sum = baseCount; - if (as != null) { - for (int i = 0; i < as.length; ++i) { - if ((a = as[i]) != null) - sum += a.value; - } - } - return sum; - } - - public interface IntEntry { - boolean equals(Object var1); - - int getKey(); - - V getValue(); - - int hashCode(); - - V setValue(V var1); - } - - /** - * Base of key, value, and entry Iterators. Adds fields to - * Traverser to support iterator.remove. - */ - static class BaseIterator extends Traverser { - Node lastReturned; - ConcurrentIntHashMap map; - - public final boolean hasNext() { - return next != null; - } - - public final void remove() { - Node p; - if ((p = lastReturned) == null) - throw new IllegalStateException(); - lastReturned = null; - map.replaceNode(p.key, null, null); - } - - void of(ConcurrentIntHashMap map) { - Node[] tab = map.table; - int l = tab == null ? 0 : tab.length; - super.of(tab, l, l); - this.map = map; - advance(); - } - } - - /** - * Base class for views. - */ - abstract static class CollectionView - implements Collection, Serializable { - private static final String oomeMsg = "Required array size too large"; - private static final long serialVersionUID = 7249069246763182397L; - final ConcurrentIntHashMap map; - - CollectionView(ConcurrentIntHashMap map) { - this.map = map; - } - - /** - * Removes all of the elements from this view, by removing all - * the mappings from the map backing this view. - */ - public final void clear() { - map.clear(); - } - - public abstract boolean contains(Object o); - - public final boolean containsAll(@NotNull Collection c) { - if (c != this) { - for (Object e : c) { - if (e == null || !contains(e)) - return false; - } - } - return true; - } - - /** - * Returns the map backing this view. - * - * @return the map backing this view - */ - public ConcurrentIntHashMap getMap() { - return map; - } - - public final boolean isEmpty() { - return map.isEmpty(); - } - - /** - * Returns an iterator over the elements in this collection. - *

    The returned iterator is - * weakly consistent. - * - * @return an iterator over the elements in this collection - */ - @NotNull - public abstract Iterator iterator(); - - public abstract boolean remove(Object o); - - @Override - public final boolean removeAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - // implementations below rely on concrete classes supplying these - // abstract methods - - @Override - public final boolean retainAll(@NotNull Collection c) { - boolean modified = false; - for (Iterator it = iterator(); it.hasNext(); ) { - if (!c.contains(it.next())) { - it.remove(); - modified = true; - } - } - return modified; - } - - @Override - public final int size() { - return map.size(); - } - - @Override - public final Object @NotNull [] toArray() { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int n = (int) sz; - Object[] r = new Object[n]; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = e; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - @Override - @SuppressWarnings("unchecked") - public final T @NotNull [] toArray(@NotNull T @NotNull [] a) { - long sz = map.mappingCount(); - if (sz > MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - int m = (int) sz; - T[] r = (a.length >= m) ? a : - (T[]) java.lang.reflect.Array - .newInstance(a.getClass().getComponentType(), m); - int n = r.length; - int i = 0; - for (E e : this) { - if (i == n) { - if (n >= MAX_ARRAY_SIZE) - throw new OutOfMemoryError(oomeMsg); - if (n >= MAX_ARRAY_SIZE - (MAX_ARRAY_SIZE >>> 1) - 1) - n = MAX_ARRAY_SIZE; - else - n += (n >>> 1) + 1; - r = Arrays.copyOf(r, n); - } - r[i++] = (T) e; - } - if (a == r && i < n) { - r[i] = null; // null-terminate - return r; - } - return (i == n) ? r : Arrays.copyOf(r, i); - } - - /** - * Returns a string representation of this collection. - * The string representation consists of the string representations - * of the collection's elements in the order they are returned by - * its iterator, enclosed in square brackets ({@code "[]"}). - * Adjacent elements are separated by the characters {@code ", "} - * (comma and space). Elements are converted to strings as by - * {@link String#valueOf(Object)}. - * - * @return a string representation of this collection - */ - @Override - public final String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('['); - Iterator it = iterator(); - if (it.hasNext()) { - for (; ; ) { - Object e = it.next(); - sb.append(e == this ? "(this Collection)" : e); - if (!it.hasNext()) - break; - sb.append(',').append(' '); - } - } - return sb.append(']').toString(); - } - } - - /** - * A padded cell for distributing counts. Adapted from LongAdder - * and Striped64. See their internal docs for explanation. - */ - static final class CounterCell { - final long value; - - CounterCell(long x) { - value = x; - } - } - - static final class EntryIterator extends BaseIterator - implements Iterator> { - - @Override - public IntEntry next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - int k = p.key; - V v = p.val; - lastReturned = p; - advance(); - return new MapEntry<>(k, v, map); - } - } - - /** - * A view of a ConcurrentLongHashMap as a {@link Set} of (key, value) - * entries. This class cannot be directly instantiated. See - * {@link #entrySet()}. - */ - static final class EntrySetView extends CollectionView> - implements Set>, Serializable { - private static final long serialVersionUID = 2249069246763182397L; - - private final ThreadLocal> tlEntryIterator = ThreadLocal.withInitial(EntryIterator::new); - - EntrySetView(ConcurrentIntHashMap map) { - super(map); - } - - @Override - public boolean add(IntEntry e) { - return map.putVal(e.getKey(), e.getValue(), false) == null; - } - - @Override - public boolean addAll(@NotNull Collection> c) { - boolean added = false; - for (IntEntry e : c) { - if (add(e)) - added = true; - } - return added; - } - - @Override - public boolean contains(Object o) { - int k; - Object v, r; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (r = map.get(k)) != null && - (v = e.getValue()) != null && - (v == r || v.equals(r))); - } - - @Override - public boolean equals(Object o) { - Set c; - return ((o instanceof Set) && - ((c = (Set) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - @Override - public int hashCode() { - int h = 0; - Node[] t = map.table; - if (t != null) { - Traverser it = map.getTraverser(t); - for (Node p; (p = it.advance()) != null; ) { - h += p.hashCode(); - } - } - return h; - } - - /** - * @return an iterator over the entries of the backing map - */ - @NotNull - public Iterator> iterator() { - EntryIterator it = tlEntryIterator.get(); - it.of(map); - return it; - } - - @Override - public boolean remove(Object o) { - int k; - Object v; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (v = e.getValue()) != null && - map.remove(k, (V) v)); - } - } - - /** - * A node inserted at head of bins during transfer operations. - */ - static final class ForwardingNode extends Node { - final Node[] nextTable; - - ForwardingNode(Node[] tab) { - super(MOVED, EMPTY_KEY, null, null); - this.nextTable = tab; - } - - @Override - Node find(int h, int k) { - // loop to avoid arbitrarily deep recursion on forwarding nodes - outer: - for (Node[] tab = nextTable; ; ) { - Node e; - int n; - if (k == EMPTY_KEY || tab == null || (n = tab.length) == 0 || - (e = tabAt(tab, (n - 1) & h)) == null) - return null; - for (; ; ) { - int eh; - if ((eh = e.hash) == h && e.key == k) - return e; - if (eh < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - continue outer; - } else - return e.find(h, k); - } - if ((e = e.next) == null) - return null; - } - } - } - } - - public static final class KeyIterator extends BaseIterator { - - public int next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - int k = p.key; - lastReturned = p; - advance(); - return k; - } - } - - /** - * A view of a ConcurrentLongHashMap as a long set of keys, in - * which additions may optionally be enabled by mapping to a - * common value. This class cannot be directly instantiated. - * See {@link #keySet() keySet()}, - * {@link #keySet(Object) keySet(V)}, - * {@link #newKeySet() newKeySet()}, - * {@link #newKeySet(int) newKeySet(int)}. - * - * @since 1.8 - */ - public static class KeySetView implements Serializable { - private static final long serialVersionUID = 7249069246763182397L; - private final ConcurrentIntHashMap map; - private final ThreadLocal> tlKeyIterator = ThreadLocal.withInitial(KeyIterator::new); - private final V value; - - KeySetView(ConcurrentIntHashMap map, V value) { // non-public - this.map = map; - this.value = value; - } - - /** - * Adds the specified key to this set view by mapping the key to - * the default mapped value in the backing map, if defined. - * - * @param k key to be added - * @return {@code true} if this set changed as a result of the call - * @throws NullPointerException if the specified key is null - * @throws UnsupportedOperationException if no default mapped value - * for additions was provided - */ - public boolean add(int k) { - V v; - if ((v = value) == null) - throw new UnsupportedOperationException(); - return map.putVal(k, v, true) == null; - } - - public final void clear() { - map.clear(); - } - - public boolean contains(int k) { - return map.containsKey(k); - } - - @Override - public boolean equals(Object o) { - KeySetView c; - return ((o instanceof KeySetView) && - ((c = (KeySetView) o) == this || - (containsAll(c) && c.containsAll(this)))); - } - - /** - * Returns the default mapped value for additions, - * or {@code null} if additions are not supported. - * - * @return the default mapped value for additions, or {@code null} - * if not supported - */ - public V getMappedValue() { - return value; - } - - @Override - public int hashCode() { - int h = 0; - KeyIterator it = iterator(); - if (it.hasNext()) { - do { - int k = it.next(); - h += keyHashCode(k); - } while (it.hasNext()); - } - return h; - } - - public final boolean isEmpty() { - return map.isEmpty(); - } - - /** - * @return an iterator over the keys of the backing map - */ - @NotNull - public KeyIterator iterator() { - KeyIterator it = tlKeyIterator.get(); - it.of(map); - return it; - } - - /** - * Removes the key from this map view, by removing the key (and its - * corresponding value) from the backing map. This method does - * nothing if the key is not in the map. - * - * @param k the key to be removed from the backing map - * @return {@code true} if the backing map contained the specified key - * @throws NullPointerException if the specified key is null - */ - public boolean remove(int k) { - return map.remove(k) != null; - } - - public final int size() { - return map.size(); - } - - @Override - public final String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('['); - KeyIterator it = iterator(); - if (it.hasNext()) { - for (; ; ) { - int k = it.next(); - sb.append(k); - if (!it.hasNext()) - break; - sb.append(',').append(' '); - } - } - return sb.append(']').toString(); - } - - private boolean containsAll(@NotNull KeySetView c) { - KeyIterator it = iterator(); - if (it.hasNext()) { - do { - int k = it.next(); - if (!contains(k)) - return false; - } while (it.hasNext()); - } - return true; - } - } - - /** - * Exported Entry for EntryIterator - */ - static final class MapEntry implements IntEntry { - final int key; // != EMPTY_KEY - final ConcurrentIntHashMap map; - V val; // non-null - - MapEntry(int key, V val, ConcurrentIntHashMap map) { - this.key = key; - this.val = val; - this.map = map; - } - - @Override - public boolean equals(Object o) { - int k; - Object v; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (v = e.getValue()) != null && - (k == key) && - (v == val || v.equals(val))); - } - - @Override - public int getKey() { - return key; - } - - @Override - public V getValue() { - return val; - } - - @Override - public int hashCode() { - return keyHashCode(key) ^ val.hashCode(); - } - - /** - * Sets our entry's value and writes through to the map. The - * value to return is somewhat arbitrary here. Since we do not - * necessarily track asynchronous changes, the most recent - * "previous" value could be different from what we return (or - * could even have been removed, in which case the put will - * re-establish). We do not and cannot guarantee more. - */ - @NotNull - public V setValue(V value) { - if (value == null) throw new NullPointerException(); - V v = val; - val = value; - map.put(key, value); - return v; - } - - @Override - public String toString() { - return key + "=" + val; - } - } - - /** - * Key-value entry. This class is never exported out as a - * user-mutable Map.Entry (i.e., one supporting setValue; see - * MapEntry below), but can be used for read-only traversals used - * in bulk tasks. Subclasses of Node with a negative hash field - * are special, and contain null keys and values (but are never - * exported). Otherwise, keys and vals are never null. - */ - static class Node implements IntEntry { - final int hash; - final int key; - volatile Node next; - volatile V val; - - Node(int hash, int key, V val, Node next) { - this.hash = hash; - this.key = key; - this.val = val; - this.next = next; - } - - @Override - public final boolean equals(Object o) { - int k; - Object v, u; - IntEntry e; - return ((o instanceof IntEntry) && - (k = (e = (IntEntry) o).getKey()) != EMPTY_KEY && - (v = e.getValue()) != null && - (k == key) && - (v == (u = val) || v.equals(u))); - } - - @Override - public final int getKey() { - return key; - } - - @Override - public final V getValue() { - return val; - } - - @Override - public final int hashCode() { - return keyHashCode(key) ^ val.hashCode(); - } - - @Override - public final V setValue(V value) { - throw new UnsupportedOperationException(); - } - - @Override - public final String toString() { - return key + "=" + val; - } - - /** - * Virtualized support for map.get(); overridden in subclasses. - */ - Node find(int h, int k) { - Node e = this; - if (k != EMPTY_KEY) { - do { - if (e.hash == h && (e.key == k)) - return e; - } while ((e = e.next) != null); - } - return null; - } - } - - /** - * A place-holder node used in computeIfAbsent and compute - */ - static final class ReservationNode extends Node { - ReservationNode() { - super(RESERVED, EMPTY_KEY, null, null); - } - - @Override - Node find(int h, int k) { - return null; - } - } - - /** - * Stripped-down version of helper class used in previous version, - * declared for the sake of serialization compatibility - */ - static class Segment extends ReentrantLock implements Serializable { - private static final long serialVersionUID = 2249069246763182397L; - final float loadFactor; - - Segment() { - this.loadFactor = ConcurrentIntHashMap.LOAD_FACTOR; - } - } - - /** - * Records the table, its length, and current traversal index for a - * traverser that must process a region of a forwarded table before - * proceeding with current table. - */ - static final class TableStack { - int index; - int length; - TableStack next; - Node[] tab; - } - - /** - * Encapsulates traversal for methods such as containsValue; also - * serves as a base class for other iterators and spliterators. - *

    - * Method advance visits once each still-valid node that was - * reachable upon iterator construction. It might miss some that - * were added to a bin after the bin was visited, which is OK wrt - * consistency guarantees. Maintaining this property in the face - * of possible ongoing resizes requires a fair amount of - * bookkeeping state that is difficult to optimize away amidst - * volatile accesses. Even so, traversal maintains reasonable - * throughput. - *

    - * Normally, iteration proceeds bin-by-bin traversing lists. - * However, if the table has been resized, then all future steps - * must traverse both the bin at the current index as well as at - * (index + baseSize); and so on for further resizings. To - * paranoically cope with potential sharing by users of iterators - * across threads, iteration terminates if a bounds checks fails - * for a table read. - */ - static class Traverser { - int baseIndex; // current index of initial table - int baseLimit; // index bound for initial table - int baseSize; // initial table size - int index; // index of bin to use next - Node next; // the next entry to use - TableStack stack, spare; // to save/restore on ForwardingNodes - Node[] tab; // current table; updated if resized - - /** - * Saves traversal state upon encountering a forwarding node. - */ - private void pushState(Node[] t, int i, int n) { - TableStack s = spare; // reuse if possible - if (s != null) - spare = s.next; - else - s = new TableStack<>(); - s.tab = t; - s.length = n; - s.index = i; - s.next = stack; - stack = s; - } - - /** - * Possibly pops traversal state. - * - * @param n length of current table - */ - private void recoverState(int n) { - TableStack s; - int len; - while ((s = stack) != null && (index += (len = s.length)) >= n) { - n = len; - index = s.index; - tab = s.tab; - s.tab = null; - TableStack next = s.next; - s.next = spare; // save for reuse - stack = next; - spare = s; - } - if (s == null && (index += baseSize) >= n) - index = ++baseIndex; - } - - /** - * Advances if possible, returning next valid node, or null if none. - */ - final Node advance() { - Node e; - if ((e = next) != null) - e = e.next; - for (; ; ) { - Node[] t; - int i, n; // must use locals in checks - if (e != null) - return next = e; - if (baseIndex >= baseLimit || (t = tab) == null || - (n = t.length) <= (i = index) || i < 0) - return next = null; - if ((e = tabAt(t, i)) != null && e.hash < 0) { - if (e instanceof ForwardingNode) { - tab = ((ForwardingNode) e).nextTable; - e = null; - pushState(t, i, n); - continue; - } else if (e instanceof TreeBin) - e = ((TreeBin) e).first; - else - e = null; - } - if (stack != null) - recoverState(n); - else if ((index = i + baseSize) >= n) - index = ++baseIndex; // visit upper slots if present - } - } - - void of(Node[] tab, int size, int limit) { - this.tab = tab; - this.baseSize = size; - this.baseIndex = this.index = 0; - this.baseLimit = limit; - this.next = null; - } - } - - /** - * TreeNodes used at the heads of bins. TreeBins do not hold user - * keys or values, but instead point to list of TreeNodes and - * their root. They also maintain a parasitic read-write lock - * forcing writers (who hold bin lock) to wait for readers (who do - * not) to complete before tree restructuring operations. - */ - static final class TreeBin extends Node { - static final int READER = 4; // increment value for setting read lock - static final int WAITER = 2; // set when waiting for write lock - // values for lockState - static final int WRITER = 1; // set while holding write lock - private static final long LOCKSTATE; - private static final sun.misc.Unsafe U; - volatile TreeNode first; - volatile int lockState; - TreeNode root; - volatile Thread waiter; - - /** - * Creates bin with initial set of nodes headed by b. - */ - TreeBin(TreeNode b) { - super(TREEBIN, EMPTY_KEY, null, null); - this.first = b; - TreeNode r = null; - for (TreeNode x = b, next; x != null; x = next) { - next = (TreeNode) x.next; - x.left = x.right = null; - if (r == null) { - x.parent = null; - x.red = false; - r = x; - } else { - int k = x.key; - int h = x.hash; - for (TreeNode p = r; ; ) { - int dir, ph; - int pk = p.key; - if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((dir = compareComparables(k, pk)) == 0) - dir = tieBreakOrder(k, pk); - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - x.parent = xp; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - r = balanceInsertion(r, x); - break; - } - } - } - } - this.root = r; - assert checkInvariants(root); - } - - /** - * Possibly blocks awaiting root lock. - */ - private void contendedLock() { - boolean waiting = false; - for (int s; ; ) { - if (((s = lockState) & ~WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, WRITER)) { - if (waiting) - waiter = null; - return; - } - } else if ((s & WAITER) == 0) { - if (U.compareAndSwapInt(this, LOCKSTATE, s, s | WAITER)) { - waiting = true; - waiter = Thread.currentThread(); - } - } else if (waiting) - LockSupport.park(this); - } - } - - /** - * Acquires write lock for tree restructuring. - */ - private void lockRoot() { - if (!U.compareAndSwapInt(this, LOCKSTATE, 0, WRITER)) - contendedLock(); // offload to separate method - } - - /** - * Releases write lock for tree restructuring. - */ - private void unlockRoot() { - lockState = 0; - } - - static TreeNode balanceDeletion(TreeNode root, - TreeNode x) { - for (TreeNode xp, xpl, xpr; ; ) { - if (x == null || x == root) - return root; - else if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (x.red) { - x.red = false; - return root; - } else if ((xpl = xp.left) == x) { - if ((xpr = xp.right) != null && xpr.red) { - xpr.red = false; - xp.red = true; - root = rotateLeft(root, xp); - xpr = (xp = x.parent) == null ? null : xp.right; - } - if (xpr == null) - x = xp; - else { - TreeNode sl = xpr.left, sr = xpr.right; - if ((sr == null || !sr.red) && - (sl == null || !sl.red)) { - xpr.red = true; - x = xp; - } else { - if (sr == null || !sr.red) { - sl.red = false; - xpr.red = true; - root = rotateRight(root, xpr); - xpr = (xp = x.parent) == null ? - null : xp.right; - } - if (xpr != null) { - xpr.red = xp.red; - if ((sr = xpr.right) != null) - sr.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateLeft(root, xp); - } - x = root; - } - } - } else { // symmetric - if (xpl != null && xpl.red) { - xpl.red = false; - xp.red = true; - root = rotateRight(root, xp); - xpl = (xp = x.parent) == null ? null : xp.left; - } - if (xpl == null) - x = xp; - else { - TreeNode sl = xpl.left, sr = xpl.right; - if ((sl == null || !sl.red) && - (sr == null || !sr.red)) { - xpl.red = true; - x = xp; - } else { - if (sl == null || !sl.red) { - sr.red = false; - xpl.red = true; - root = rotateLeft(root, xpl); - xpl = (xp = x.parent) == null ? - null : xp.left; - } - if (xpl != null) { - xpl.red = xp.red; - if ((sl = xpl.left) != null) - sl.red = false; - } - if (xp != null) { - xp.red = false; - root = rotateRight(root, xp); - } - x = root; - } - } - } - } - } - - static TreeNode balanceInsertion(TreeNode root, - TreeNode x) { - x.red = true; - for (TreeNode xp, xpp, xppl, xppr; ; ) { - if ((xp = x.parent) == null) { - x.red = false; - return x; - } else if (!xp.red || (xpp = xp.parent) == null) - return root; - if (xp == (xppl = xpp.left)) { - if ((xppr = xpp.right) != null && xppr.red) { - xppr.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.right) { - root = rotateLeft(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateRight(root, xpp); - } - } - } - } else { - if (xppl != null && xppl.red) { - xppl.red = false; - xp.red = false; - xpp.red = true; - x = xpp; - } else { - if (x == xp.left) { - root = rotateRight(root, x = xp); - xpp = (xp = x.parent) == null ? null : xp.parent; - } - if (xp != null) { - xp.red = false; - if (xpp != null) { - xpp.red = true; - root = rotateLeft(root, xpp); - } - } - } - } - } - } - - /** - * Recursive invariant check - */ - @SuppressWarnings("SimplifiableIfStatement") - static boolean checkInvariants(TreeNode t) { - TreeNode tp = t.parent, tl = t.left, tr = t.right, - tb = t.prev, tn = (TreeNode) t.next; - if (tb != null && tb.next != t) - return false; - if (tn != null && tn.prev != t) - return false; - if (tp != null && t != tp.left && t != tp.right) - return false; - if (tl != null && (tl.parent != t || tl.hash > t.hash)) - return false; - if (tr != null && (tr.parent != t || tr.hash < t.hash)) - return false; - if (t.red && tl != null && tl.red && tr != null && tr.red) - return false; - if (tl != null && !checkInvariants(tl)) - return false; - return !(tr != null && !checkInvariants(tr)); - } - - static TreeNode rotateLeft(TreeNode root, TreeNode p) { - TreeNode r, pp, rl; - if (p != null && (r = p.right) != null) { - if ((rl = p.right = r.left) != null) - rl.parent = p; - if ((pp = r.parent = p.parent) == null) - (root = r).red = false; - else if (pp.left == p) - pp.left = r; - else - pp.right = r; - r.left = p; - p.parent = r; - } - return root; - } - - static TreeNode rotateRight(TreeNode root, TreeNode p) { - TreeNode l, pp, lr; - if (p != null && (l = p.left) != null) { - if ((lr = p.left = l.right) != null) - lr.parent = p; - if ((pp = l.parent = p.parent) == null) - (root = l).red = false; - else if (pp.right == p) - pp.right = l; - else - pp.left = l; - l.right = p; - p.parent = l; - } - return root; - } - - /** - * Tie-breaking utility for ordering insertions when equal - * hashCodes and non-comparable. We don't require a total - * order, just a consistent insertion rule to maintain - * equivalence across rebalancings. Tie-breaking further than - * necessary simplifies testing a bit. - */ - static int tieBreakOrder(Object a, Object b) { - int d; - if (a == null || b == null || - (d = a.getClass().getName(). - compareTo(b.getClass().getName())) == 0) - d = (System.identityHashCode(a) <= System.identityHashCode(b) ? - -1 : 1); - return d; - } - - /** - * Returns matching node or null if none. Tries to search - * using tree comparisons from root, but continues linear - * search when lock not available. - */ - @Override - Node find(int h, int k) { - if (k != EMPTY_KEY) { - for (Node e = first; e != null; ) { - int s; - if (((s = lockState) & (WAITER | WRITER)) != 0) { - if (e.hash == h && e.key == k) - return e; - e = e.next; - } else if (U.compareAndSwapInt(this, LOCKSTATE, s, - s + READER)) { - TreeNode r, p; - try { - p = ((r = root) == null ? null : - r.findTreeNode(h, k)); - } finally { - Thread w; - if (U.getAndAddInt(this, LOCKSTATE, -READER) == - (READER | WAITER) && (w = waiter) != null) - LockSupport.unpark(w); - } - return p; - } - } - } - return null; - } - - /** - * Finds or adds a node. - * - * @return null if added - */ - TreeNode putTreeVal(int h, int k, V v) { - boolean searched = false; - for (TreeNode p = root; ; ) { - int dir, ph; - int pk; - if (p == null) { - first = root = new TreeNode<>(h, k, v, null, null); - break; - } else if ((ph = p.hash) > h) - dir = -1; - else if (ph < h) - dir = 1; - else if ((pk = p.key) == k) - return p; - else if ((dir = compareComparables(k, pk)) == 0) { - if (!searched) { - TreeNode q, ch; - searched = true; - if (((ch = p.left) != null && - (q = ch.findTreeNode(h, k)) != null) || - ((ch = p.right) != null && - (q = ch.findTreeNode(h, k)) != null)) - return q; - } - dir = tieBreakOrder(k, pk); - } - - TreeNode xp = p; - if ((p = (dir <= 0) ? p.left : p.right) == null) { - TreeNode x, f = first; - first = x = new TreeNode<>(h, k, v, f, xp); - if (f != null) - f.prev = x; - if (dir <= 0) - xp.left = x; - else - xp.right = x; - if (!xp.red) - x.red = true; - else { - lockRoot(); - try { - root = balanceInsertion(root, x); - } finally { - unlockRoot(); - } - } - break; - } - } - assert checkInvariants(root); - return null; - } - - /** - * Removes the given node, that must be present before this - * call. This is messier than typical red-black deletion code - * because we cannot swap the contents of an interior node - * with a leaf successor that is pinned by "next" pointers - * that are accessible independently of lock. So instead we - * swap the tree linkages. - * - * @return true if now too small, so should be untreeified - */ - boolean removeTreeNode(TreeNode p) { - TreeNode next = (TreeNode) p.next; - TreeNode pred = p.prev; // unlink traversal pointers - TreeNode r, rl; - if (pred == null) - first = next; - else - pred.next = next; - if (next != null) - next.prev = pred; - if (first == null) { - root = null; - return true; - } - if ((r = root) == null || r.right == null || // too small - (rl = r.left) == null || rl.left == null) - return true; - lockRoot(); - try { - TreeNode replacement; - TreeNode pl = p.left; - TreeNode pr = p.right; - if (pl != null && pr != null) { - TreeNode s = pr, sl; - while ((sl = s.left) != null) // find successor - s = sl; - boolean c = s.red; - s.red = p.red; - p.red = c; // swap colors - TreeNode sr = s.right; - TreeNode pp = p.parent; - if (s == pr) { // p was s's direct parent - p.parent = s; - s.right = p; - } else { - TreeNode sp = s.parent; - if ((p.parent = sp) != null) { - if (s == sp.left) - sp.left = p; - else - sp.right = p; - } - s.right = pr; - pr.parent = s; - } - p.left = null; - if ((p.right = sr) != null) - sr.parent = p; - s.left = pl; - pl.parent = s; - if ((s.parent = pp) == null) - r = s; - else if (p == pp.left) - pp.left = s; - else - pp.right = s; - if (sr != null) - replacement = sr; - else - replacement = p; - } else if (pl != null) - replacement = pl; - else if (pr != null) - replacement = pr; - else - replacement = p; - if (replacement != p) { - TreeNode pp = replacement.parent = p.parent; - if (pp == null) - r = replacement; - else if (p == pp.left) - pp.left = replacement; - else - pp.right = replacement; - p.left = p.right = p.parent = null; - } - - root = (p.red) ? r : balanceDeletion(r, replacement); - - if (p == replacement) { // detach pointers - TreeNode pp; - if ((pp = p.parent) != null) { - if (p == pp.left) - pp.left = null; - else if (p == pp.right) - pp.right = null; - p.parent = null; - } - } - } finally { - unlockRoot(); - } - assert checkInvariants(root); - return false; - } - - static { - try { - U = Unsafe.getUnsafe(); - Class k = TreeBin.class; - LOCKSTATE = U.objectFieldOffset - (k.getDeclaredField("lockState")); - } catch (Exception e) { - throw new Error(e); - } - } - - - - - - - - - - - - - - - - /* ------------------------------------------------------------ */ - // Red-black tree methods, all adapted from CLR - } - - /** - * Nodes for use in TreeBins - */ - static final class TreeNode extends Node { - TreeNode left; - TreeNode parent; // red-black tree links - TreeNode prev; // needed to unlink next upon deletion - boolean red; - TreeNode right; - - TreeNode(int hash, int key, V val, Node next, TreeNode parent) { - super(hash, key, val, next); - this.parent = parent; - } - - @Override - Node find(int h, int k) { - return findTreeNode(h, k); - } - - /** - * Returns the TreeNode (or null if not found) for the given key - * starting at given root. - */ - TreeNode findTreeNode(int h, int k) { - if (k != EMPTY_KEY) { - TreeNode p = this; - do { - int ph, dir; - int pk; - TreeNode q; - TreeNode pl = p.left, pr = p.right; - if ((ph = p.hash) > h) - p = pl; - else if (ph < h) - p = pr; - else if ((pk = p.key) == k) - return p; - else if (pl == null) - p = pr; - else if (pr == null) - p = pl; - else if ((dir = compareComparables(k, pk)) != 0) - p = (dir < 0) ? pl : pr; - else if ((q = pr.findTreeNode(h, k)) != null) - return q; - else - p = pl; - } while (p != null); - } - return null; - } - - - } - - static final class ValueIterator extends BaseIterator - implements Iterator { - public V next() { - Node p; - if ((p = next) == null) - throw new NoSuchElementException(); - V v = p.val; - lastReturned = p; - advance(); - return v; - } - } - - /** - * A view of a ConcurrentLongHashMap as a {@link Collection} of - * values, in which additions are disabled. This class cannot be - * directly instantiated. See {@link #values()}. - */ - static final class ValuesView extends CollectionView - implements Collection, Serializable { - private static final long serialVersionUID = 2249069246763182397L; - private final ThreadLocal> tlValueIterator = ThreadLocal.withInitial(ValueIterator::new); - - ValuesView(ConcurrentIntHashMap map) { - super(map); - } - - @Override - public boolean add(V e) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean addAll(@NotNull Collection c) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean contains(Object o) { - return map.containsValue((V) o); - } - - @Override - @NotNull - public Iterator iterator() { - ValueIterator it = tlValueIterator.get(); - it.of(map); - return it; - } - - @Override - public boolean remove(Object o) { - if (o != null) { - for (Iterator it = iterator(); it.hasNext(); ) { - if (o.equals(it.next())) { - it.remove(); - return true; - } - } - } - return false; - } - } - - static { - try { - Class tk = Thread.class; - SEED = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomSeed")); - PROBE = Unsafe.getUnsafe().objectFieldOffset(tk.getDeclaredField("threadLocalRandomProbe")); - } catch (Exception e) { - throw new Error(e); - } - } - - static { - try { - Class k = ConcurrentIntHashMap.class; - SIZECTL = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("sizeCtl")); - TRANSFERINDEX = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("transferIndex")); - BASECOUNT = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("baseCount")); - CELLSBUSY = Unsafe.getUnsafe().objectFieldOffset - (k.getDeclaredField("cellsBusy")); - Class ck = CounterCell.class; - CELLVALUE = Unsafe.getUnsafe().objectFieldOffset - (ck.getDeclaredField("value")); - Class ak = Node[].class; - ABASE = Unsafe.getUnsafe().arrayBaseOffset(ak); - int scale = Unsafe.getUnsafe().arrayIndexScale(ak); - if ((scale & (scale - 1)) != 0) - throw new Error("data type scale not a power of two"); - ASHIFT = 31 - Integer.numberOfLeadingZeros(scale); - } catch (Exception e) { - throw new Error(e); - } - } -} diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java deleted file mode 100644 index 59b0582..0000000 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ /dev/null @@ -1,33 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -public interface FilesFacade { - boolean close(long fd); - - int errno(); - - long length(long fd); -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/GenericLexer.java b/core/src/main/java/io/questdb/client/std/GenericLexer.java deleted file mode 100644 index 8a7bedd..0000000 --- a/core/src/main/java/io/questdb/client/std/GenericLexer.java +++ /dev/null @@ -1,435 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import io.questdb.client.std.str.AbstractCharSequence; -import io.questdb.client.std.str.Utf16Sink; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.ArrayDeque; -import java.util.Comparator; - -public class GenericLexer implements ImmutableIterator, Mutable { - public static final LenComparator COMPARATOR = new LenComparator(); - public static final CharSequenceHashSet WHITESPACE = new CharSequenceHashSet(); - public static final IntHashSet WHITESPACE_CH = new IntHashSet(); - - private final ObjectPool csPairPool; - private final ObjectPool csPool; - private final ObjectPool csTriplePool; - private final CharSequence flyweightSequence = new InternalFloatingSequence(); - private final IntStack stashedNumbers = new IntStack(); - private final ArrayDeque stashedStrings = new ArrayDeque<>(); - private final IntObjHashMap> symbols = new IntObjHashMap<>(); - private final ArrayDeque unparsed = new ArrayDeque<>(); - private final IntStack unparsedPosition = new IntStack(); - private int _hi; - private int _len; - private int _lo; - private int _pos; - - private CharSequence content; - private CharSequence last; - private CharSequence next = null; - - public GenericLexer(int poolCapacity) { - csPool = new ObjectPool<>(FloatingSequence::new, poolCapacity); - csPairPool = new ObjectPool<>(FloatingSequencePair::new, poolCapacity); - csTriplePool = new ObjectPool<>(FloatingSequenceTriple::new, poolCapacity); - for (int i = 0, n = WHITESPACE.size(); i < n; i++) { - defineSymbol(Chars.toString(WHITESPACE.get(i))); - } - } - - @Override - public void clear() { - of(null, 0, 0); - - stashedNumbers.clear(); - stashedStrings.clear(); - } - - public final void defineSymbol(String token) { - char c0 = token.charAt(0); - ObjList l; - int index = symbols.keyIndex(c0); - if (index > -1) { - l = new ObjList<>(); - symbols.putAt(index, c0, l); - } else { - l = symbols.valueAtQuick(index); - } - l.add(token); - l.sort(COMPARATOR); - } - - @Override - public boolean hasNext() { - boolean n = next != null || hasUnparsed() || (content != null && _pos < _len); - if (!n && last != null) { - last = null; - } - return n; - } - - public boolean hasUnparsed() { - return !unparsed.isEmpty(); - } - - @Override - public CharSequence next() { - if (!unparsed.isEmpty()) { - this._lo = unparsedPosition.pollLast(); - this._pos = unparsedPosition.pollLast(); - - return last = unparsed.pollLast(); - } - - this._lo = this._hi; - - if (next != null) { - CharSequence result = next; - next = null; - return last = result; - } - - this._lo = this._hi = _pos; - - char term = 0; - int openTermIdx = -1; - while (_pos < _len) { - char c = content.charAt(_pos++); - CharSequence token; - switch (term) { - case 0: - switch (c) { - case '\'': - term = '\''; - openTermIdx = _pos - 1; - break; - case '"': - term = '"'; - openTermIdx = _pos - 1; - break; - case '`': - term = '`'; - openTermIdx = _pos - 1; - break; - default: - if ((token = token(c)) != null) { - return last = token; - } else { - _hi++; - } - break; - } - break; - case '\'': - if (c == '\'') { - _hi += 2; - if (_pos < _len && content.charAt(_pos) == '\'') { - _pos++; - } else { - return last = flyweightSequence; - } - } else { - _hi++; - } - break; - case '"': - if (c == '"') { - _hi += 2; - if (_pos < _len && content.charAt(_pos) == '"') { - _pos++; - } else { - return last = flyweightSequence; - } - } else { - _hi++; - } - break; - case '`': - if (c == '`') { - _hi += 2; - return last = flyweightSequence; - } else { - _hi++; - } - break; - default: - break; - } - } - if (openTermIdx != -1) { // dangling terms - if (_len == 1) { - _hi += 1; // emit term - } else { - if (openTermIdx == _lo) { // term is at the start - _hi = _lo + 1; // emit term - _pos = _hi; // rewind pos - } else if (openTermIdx == _len - 1) { // term is at the end, high is right on term - FloatingSequence termFs = csPool.next(); - termFs.lo = _hi; - termFs.hi = _hi + 1; - next = termFs; // emit term next - } else { // term is somewhere in between - _hi = openTermIdx; // emit whatever comes before term - _pos = openTermIdx; // rewind pos - } - } - } - return last = flyweightSequence; - } - - public void of(CharSequence cs, int lo, int hi) { - this.csPool.clear(); - this.csPairPool.clear(); - this.csTriplePool.clear(); - this.content = cs; - this._pos = lo; - this._len = hi; - this.next = null; - this.unparsed.clear(); - this.unparsedPosition.clear(); - this.last = null; - } - - private static CharSequence findToken0(char c, CharSequence content, int _pos, int _len, IntObjHashMap> symbols) { - final int index = symbols.keyIndex(c); - return index > -1 ? null : findToken00(content, _pos, _len, symbols, index); - } - - @Nullable - private static CharSequence findToken00(CharSequence content, int _pos, int _len, IntObjHashMap> symbols, int index) { - final ObjList l = symbols.valueAt(index); - for (int i = 0, sz = l.size(); i < sz; i++) { - CharSequence txt = l.getQuick(i); - int n = txt.length(); - boolean match = (n - 2) < (_len - _pos); - if (match) { - for (int k = 1; k < n; k++) { - if (content.charAt(_pos + (k - 1)) != txt.charAt(k)) { - match = false; - break; - } - } - } - - if (match) { - return txt; - } - } - return null; - } - - private CharSequence token(char c) { - CharSequence t = findToken0(c, content, _pos, _len, symbols); - if (t != null) { - _pos = _pos + t.length() - 1; - if (_lo == _hi) { - return t; - } - next = t; - return flyweightSequence; - } else { - return null; - } - } - - public static class FloatingSequencePair extends AbstractCharSequence implements Mutable { - public static final char NO_SEPARATOR = (char) 0; - - public FloatingSequence cs0; - public FloatingSequence cs1; - char sep = NO_SEPARATOR; - - @Override - public char charAt(int index) { - int cs0Len = cs0.length(); - if (index < cs0Len) { - return cs0.charAt(index); - } - if (sep == NO_SEPARATOR) { - return cs1.charAt(index - cs0Len); - } - return index == cs0Len ? sep : cs1.charAt(index - cs0Len - 1); - } - - @Override - public void clear() { - // no-op - } - - @Override - public int length() { - return cs0.length() + cs1.length() + (sep != NO_SEPARATOR ? 1 : 0); - } - - @NotNull - @Override - public String toString() { - final Utf16Sink b = Misc.getThreadLocalSink(); - b.put(cs0); - if (sep != NO_SEPARATOR) { - b.put(sep); - } - b.put(cs1); - return b.toString(); - } - } - - public static class FloatingSequenceTriple extends AbstractCharSequence implements Mutable { - public static final char NO_SEPARATOR = (char) 0; - - public FloatingSequence cs0; - public FloatingSequence cs1; - public FloatingSequence cs2; - char sep = NO_SEPARATOR; - - @Override - public char charAt(int index) { - int cs0Len = cs0.length(); - if (index < cs0Len) { - return cs0.charAt(index); - } - index -= cs0Len; - if (sep != NO_SEPARATOR) { - if (index == 0) { - return sep; - } - index--; - } - int cs1Len = cs1.length(); - if (index < cs1Len) { - return cs1.charAt(index); - } - index -= cs1Len; - if (sep != NO_SEPARATOR) { - if (index == 0) { - return sep; - } - index--; - } - return cs2.charAt(index); - } - - @Override - public void clear() { - // no-op - } - - @Override - public int length() { - return cs0.length() + cs1.length() + cs2.length() + (sep != NO_SEPARATOR ? 2 : 0); - } - - @NotNull - @Override - public String toString() { - final Utf16Sink b = Misc.getThreadLocalSink(); - b.put(cs0); - if (sep != NO_SEPARATOR) { - b.put(sep); - } - b.put(cs1); - if (sep != NO_SEPARATOR) { - b.put(sep); - } - b.put(cs2); - return b.toString(); - } - } - - public static class LenComparator implements Comparator { - @Override - public int compare(CharSequence o1, CharSequence o2) { - return o2.length() - o1.length(); - } - } - - public class FloatingSequence extends AbstractCharSequence implements Mutable, BufferWindowCharSequence { - int hi; - int lo; - - @Override - public char charAt(int index) { - return content.charAt(lo + index); - } - - @Override - public void clear() { - } - - @Override - public int length() { - return hi - lo; - } - - @Override - protected final CharSequence _subSequence(int start, int end) { - FloatingSequence that = csPool.next(); - that.lo = lo + start; - that.hi = lo + end; - assert that.lo <= that.hi; - return that; - } - } - - public class InternalFloatingSequence extends AbstractCharSequence { - - @Override - public char charAt(int index) { - return content.charAt(_lo + index); - } - - @Override - public int length() { - return _hi - _lo; - } - - @Override - protected CharSequence _subSequence(int start, int end) { - FloatingSequence next = csPool.next(); - next.lo = _lo + start; - next.hi = _lo + end; - assert next.lo <= next.hi; - return next; - } - - } - - static { - WHITESPACE.add(" "); - WHITESPACE.add("\t"); - WHITESPACE.add("\n"); - WHITESPACE.add("\r"); - - WHITESPACE_CH.add(' '); - WHITESPACE_CH.add('\t'); - WHITESPACE_CH.add('\n'); - WHITESPACE_CH.add('\r'); - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/LongObjHashMap.java b/core/src/main/java/io/questdb/client/std/LongObjHashMap.java deleted file mode 100644 index 09a7ef6..0000000 --- a/core/src/main/java/io/questdb/client/std/LongObjHashMap.java +++ /dev/null @@ -1,110 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import java.util.Arrays; - -public class LongObjHashMap extends AbstractLongHashSet { - private V[] values; - - public LongObjHashMap() { - this(8); - } - - public LongObjHashMap(int initialCapacity) { - this(initialCapacity, 0.5f); - } - - @SuppressWarnings("unchecked") - private LongObjHashMap(int initialCapacity, double loadFactor) { - super(initialCapacity, loadFactor); - values = (V[]) new Object[keys.length]; - clear(); - } - - @Override - public void clear() { - super.clear(); - Arrays.fill(values, null); - } - - public void putAt(int index, long key, V value) { - if (index < 0) { - values[-index - 1] = value; - } else { - keys[index] = key; - values[index] = value; - if (--free == 0) { - rehash(); - } - } - } - - public V valueAt(int index) { - return index < 0 ? valueAtQuick(index) : null; - } - - public V valueAtQuick(int index) { - return values[-index - 1]; - } - - @SuppressWarnings("unchecked") - private void rehash() { - int size = size(); - int newCapacity = capacity * 2; - free = capacity = newCapacity; - int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); - - V[] oldValues = values; - long[] oldKeys = keys; - this.keys = new long[len]; - this.values = (V[]) new Object[len]; - Arrays.fill(keys, noEntryKeyValue); - mask = len - 1; - - free -= size; - for (int i = oldKeys.length; i-- > 0; ) { - long key = oldKeys[i]; - if (key != noEntryKeyValue) { - final int index = keyIndex(key); - keys[index] = key; - values[index] = oldValues[i]; - } - } - } - - @Override - protected void erase(int index) { - keys[index] = this.noEntryKeyValue; - } - - @Override - protected void move(int from, int to) { - keys[to] = keys[from]; - values[to] = values[from]; - erase(from); - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/LowerCaseCharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/LowerCaseCharSequenceHashSet.java deleted file mode 100644 index 40d6090..0000000 --- a/core/src/main/java/io/questdb/client/std/LowerCaseCharSequenceHashSet.java +++ /dev/null @@ -1,106 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -public class LowerCaseCharSequenceHashSet extends AbstractLowerCaseCharSequenceHashSet { - private static final int MIN_INITIAL_CAPACITY = 16; - - public LowerCaseCharSequenceHashSet() { - this(MIN_INITIAL_CAPACITY); - } - - private LowerCaseCharSequenceHashSet(int initialCapacity) { - this(initialCapacity, 0.4); - } - - private LowerCaseCharSequenceHashSet(int initialCapacity, double loadFactor) { - super(initialCapacity, loadFactor); - clear(); - } - - /** - * Adds key to hash set preserving key uniqueness. - * - * @param key immutable sequence of characters. - * @return false if key is already in the set and true otherwise. - */ - public boolean add(CharSequence key) { - int index = keyIndex(key); - if (index < 0) { - return false; - } - - addAt(index, key); - return true; - } - - public void addAt(int index, CharSequence key) { - keys[index] = key; - if (--free < 1) { - rehash(); - } - } - - // returns the first non-null key, in arbitrary order - public CharSequence getAny() { - for (int i = 0, n = keys.length; i < n; i++) { - if (keys[i] != noEntryKey) { - return keys[i]; - } - } - return null; - } - - public CharSequence keyAt(int index) { - return keys[-index - 1]; - } - - private void rehash() { - int newCapacity = capacity * 2; - final int size = size(); - free = capacity = newCapacity; - int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); - CharSequence[] newKeys = new CharSequence[len]; - CharSequence[] oldKeys = keys; - mask = len - 1; - this.keys = newKeys; - free -= size; - for (int i = 0, n = oldKeys.length; i < n; i++) { - CharSequence key = oldKeys[i]; - if (key != null) { - keys[keyIndex(key)] = key; - } - } - } - - protected void erase(int index) { - keys[index] = noEntryKey; - } - - protected void move(int from, int to) { - keys[to] = keys[from]; - erase(from); - } -} diff --git a/core/src/main/java/io/questdb/client/std/ex/BytecodeException.java b/core/src/main/java/io/questdb/client/std/ex/BytecodeException.java deleted file mode 100644 index cff0ea5..0000000 --- a/core/src/main/java/io/questdb/client/std/ex/BytecodeException.java +++ /dev/null @@ -1,33 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std.ex; - -public class BytecodeException extends RuntimeException { - public static final BytecodeException INSTANCE = new BytecodeException(); - - private BytecodeException() { - super("Error in bytecode"); - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/str/DirectCharSequence.java b/core/src/main/java/io/questdb/client/std/str/DirectCharSequence.java deleted file mode 100644 index fcbecb1..0000000 --- a/core/src/main/java/io/questdb/client/std/str/DirectCharSequence.java +++ /dev/null @@ -1,33 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std.str; - -import io.questdb.client.std.bytes.DirectSequence; - -/** - * A sequence of UTF-16 chars stored in native memory. - */ -public interface DirectCharSequence extends CharSequence, DirectSequence { -} \ No newline at end of file diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java deleted file mode 100644 index 2a7491d..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java +++ /dev/null @@ -1,316 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.protocol; - -import io.questdb.client.cutlass.qwp.protocol.QwpNullBitmap; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; -import org.junit.Assert; -import org.junit.Test; - -public class QwpNullBitmapTest { - - @Test - public void testAllNulls() throws Exception { - assertMemoryLeak(() -> { - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillAllNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); - Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); - - for (int i = 0; i < rowCount; i++) { - Assert.assertTrue(QwpNullBitmap.isNull(address, i)); - } - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testAllNullsPartialByte() throws Exception { - assertMemoryLeak(() -> { - // Test with row count not divisible by 8 - int rowCount = 10; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillAllNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); - Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testBitmapBitOrder() throws Exception { - assertMemoryLeak(() -> { - // Test LSB-first bit ordering - int rowCount = 8; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set bit 0 (LSB) - QwpNullBitmap.setNull(address, 0); - byte b = Unsafe.getUnsafe().getByte(address); - Assert.assertEquals(0b00000001, b & 0xFF); - - // Set bit 7 (MSB of first byte) - QwpNullBitmap.setNull(address, 7); - b = Unsafe.getUnsafe().getByte(address); - Assert.assertEquals(0b10000001, b & 0xFF); - - // Set bit 3 - QwpNullBitmap.setNull(address, 3); - b = Unsafe.getUnsafe().getByte(address); - Assert.assertEquals(0b10001001, b & 0xFF); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testBitmapByteAlignment() throws Exception { - assertMemoryLeak(() -> { - // Test that bits 8-15 go into second byte - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set bit 8 (first bit of second byte) - QwpNullBitmap.setNull(address, 8); - Assert.assertEquals(0, Unsafe.getUnsafe().getByte(address) & 0xFF); - Assert.assertEquals(0b00000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); - - // Set bit 15 (last bit of second byte) - QwpNullBitmap.setNull(address, 15); - Assert.assertEquals(0b10000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testBitmapSizeCalculation() throws Exception { - assertMemoryLeak(() -> { - Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); - Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(1)); - Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(7)); - Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(8)); - Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(9)); - Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(16)); - Assert.assertEquals(3, QwpNullBitmap.sizeInBytes(17)); - Assert.assertEquals(125, QwpNullBitmap.sizeInBytes(1000)); - Assert.assertEquals(125000, QwpNullBitmap.sizeInBytes(1000000)); - }); - } - - @Test - public void testBitmapWithPartialLastByte() throws Exception { - assertMemoryLeak(() -> { - // 10 rows = 2 bytes, but only 2 bits used in second byte - int rowCount = 10; - int size = QwpNullBitmap.sizeInBytes(rowCount); - Assert.assertEquals(2, size); - - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set row 9 (bit 1 of second byte) - QwpNullBitmap.setNull(address, 9); - Assert.assertTrue(QwpNullBitmap.isNull(address, 9)); - Assert.assertEquals(0b00000010, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); - - Assert.assertEquals(1, QwpNullBitmap.countNulls(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testByteArrayOperations() throws Exception { - assertMemoryLeak(() -> { - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - byte[] bitmap = new byte[size]; - int offset = 0; - - QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); - - QwpNullBitmap.setNull(bitmap, offset, 0); - QwpNullBitmap.setNull(bitmap, offset, 5); - QwpNullBitmap.setNull(bitmap, offset, 15); - - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 0)); - Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 1)); - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 5)); - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 15)); - - Assert.assertEquals(3, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); - - QwpNullBitmap.clearNull(bitmap, offset, 5); - Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 5)); - Assert.assertEquals(2, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); - }); - } - - @Test - public void testByteArrayWithOffset() throws Exception { - assertMemoryLeak(() -> { - int rowCount = 8; - int size = QwpNullBitmap.sizeInBytes(rowCount); - byte[] bitmap = new byte[10 + size]; // Extra padding - int offset = 5; // Start at offset 5 - - QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); - QwpNullBitmap.setNull(bitmap, offset, 3); - - Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 3)); - Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 4)); - }); - } - - @Test - public void testClearNull() throws Exception { - assertMemoryLeak(() -> { - int rowCount = 8; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillAllNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.isNull(address, 3)); - - QwpNullBitmap.clearNull(address, 3); - Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); - Assert.assertEquals(7, QwpNullBitmap.countNulls(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEmptyBitmap() throws Exception { - assertMemoryLeak(() -> { - Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); - }); - } - - @Test - public void testLargeBitmap() throws Exception { - assertMemoryLeak(() -> { - int rowCount = 100000; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set every 100th row as null - int expectedNulls = 0; - for (int i = 0; i < rowCount; i += 100) { - QwpNullBitmap.setNull(address, i); - expectedNulls++; - } - - Assert.assertEquals(expectedNulls, QwpNullBitmap.countNulls(address, rowCount)); - - // Verify some random positions - Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 100)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 99900)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 99)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testMixedNulls() throws Exception { - assertMemoryLeak(() -> { - int rowCount = 20; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - - // Set specific rows as null: 0, 2, 5, 19 - QwpNullBitmap.setNull(address, 0); - QwpNullBitmap.setNull(address, 2); - QwpNullBitmap.setNull(address, 5); - QwpNullBitmap.setNull(address, 19); - - Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 2)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); - Assert.assertFalse(QwpNullBitmap.isNull(address, 4)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 5)); - Assert.assertTrue(QwpNullBitmap.isNull(address, 19)); - - Assert.assertEquals(4, QwpNullBitmap.countNulls(address, rowCount)); - Assert.assertFalse(QwpNullBitmap.allNull(address, rowCount)); - Assert.assertFalse(QwpNullBitmap.noneNull(address, rowCount)); - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testNoNulls() throws Exception { - assertMemoryLeak(() -> { - int rowCount = 16; - int size = QwpNullBitmap.sizeInBytes(rowCount); - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - try { - QwpNullBitmap.fillNoneNull(address, rowCount); - Assert.assertTrue(QwpNullBitmap.noneNull(address, rowCount)); - Assert.assertEquals(0, QwpNullBitmap.countNulls(address, rowCount)); - - for (int i = 0; i < rowCount; i++) { - Assert.assertFalse(QwpNullBitmap.isNull(address, i)); - } - } finally { - Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); - } - }); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java deleted file mode 100644 index 6a7ed3b..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.protocol; - -import io.questdb.client.cutlass.qwp.protocol.QwpZigZag; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Random; - -public class QwpZigZagTest { - - @Test - public void testEncodeDecodeInt() { - // Test 32-bit version - Assert.assertEquals(0, QwpZigZag.encode(0)); - Assert.assertEquals(0, QwpZigZag.decode(0)); - - Assert.assertEquals(2, QwpZigZag.encode(1)); - Assert.assertEquals(1, QwpZigZag.decode(2)); - - Assert.assertEquals(1, QwpZigZag.encode(-1)); - Assert.assertEquals(-1, QwpZigZag.decode(1)); - - int minInt = Integer.MIN_VALUE; - int encoded = QwpZigZag.encode(minInt); - Assert.assertEquals(minInt, QwpZigZag.decode(encoded)); - - int maxInt = Integer.MAX_VALUE; - encoded = QwpZigZag.encode(maxInt); - Assert.assertEquals(maxInt, QwpZigZag.decode(encoded)); - } - - @Test - public void testEncodeDecodeZero() { - Assert.assertEquals(0, QwpZigZag.encode(0L)); - Assert.assertEquals(0, QwpZigZag.decode(0L)); - } - - @Test - public void testEncodeMaxLong() { - long encoded = QwpZigZag.encode(Long.MAX_VALUE); - Assert.assertEquals(-2L, encoded); // 0xFFFFFFFFFFFFFFFE (all bits except LSB) - Assert.assertEquals(Long.MAX_VALUE, QwpZigZag.decode(encoded)); - } - - @Test - public void testEncodeMinLong() { - long encoded = QwpZigZag.encode(Long.MIN_VALUE); - Assert.assertEquals(-1L, encoded); // All bits set (unsigned max) - Assert.assertEquals(Long.MIN_VALUE, QwpZigZag.decode(encoded)); - } - - @Test - public void testEncodeNegative() { - // ZigZag encoding maps: - // -1 -> 1 - // -2 -> 3 - // -n -> 2n - 1 - Assert.assertEquals(1, QwpZigZag.encode(-1L)); - Assert.assertEquals(3, QwpZigZag.encode(-2L)); - Assert.assertEquals(5, QwpZigZag.encode(-3L)); - Assert.assertEquals(199, QwpZigZag.encode(-100L)); - } - - @Test - public void testEncodePositive() { - // ZigZag encoding maps: - // 0 -> 0 - // 1 -> 2 - // 2 -> 4 - // n -> 2n - Assert.assertEquals(2, QwpZigZag.encode(1L)); - Assert.assertEquals(4, QwpZigZag.encode(2L)); - Assert.assertEquals(6, QwpZigZag.encode(3L)); - Assert.assertEquals(200, QwpZigZag.encode(100L)); - } - - @Test - public void testEncodingPattern() { - // Verify the exact encoding pattern matches the formula: - // zigzag(n) = (n << 1) ^ (n >> 63) - // This means: - // - Non-negative n: zigzag(n) = 2 * n - // - Negative n: zigzag(n) = -2 * n - 1 - - for (int n = -100; n <= 100; n++) { - long encoded = QwpZigZag.encode((long) n); - long expected = (n >= 0) ? (2L * n) : (-2L * n - 1); - Assert.assertEquals("Encoding mismatch for n=" + n, expected, encoded); - } - } - - @Test - public void testRoundTripRandomValues() { - Random random = new Random(42); // Fixed seed for reproducibility - - for (int i = 0; i < 1000; i++) { - long value = random.nextLong(); - long encoded = QwpZigZag.encode(value); - long decoded = QwpZigZag.decode(encoded); - Assert.assertEquals("Failed for value: " + value, value, decoded); - } - } - - @Test - public void testSmallValuesHaveSmallEncodings() { - // The point of ZigZag is that small absolute values produce small encoded values - // which then encode efficiently as varints - - // -1 encodes to 1 (small, 1 byte as varint) - Assert.assertTrue(QwpZigZag.encode(-1L) < 128); - - // Small positive and negative values should encode to small values - // Values in [-63, 63] all encode to values < 128 (1 byte varint) - // 63 encodes to 126, -63 encodes to 125 - for (int n = -63; n <= 63; n++) { - long encoded = QwpZigZag.encode(n); - Assert.assertTrue("Value " + n + " encoded to " + encoded, - encoded < 128); // Fits in 1 byte varint - } - - // 64 encodes to 128, which requires 2 bytes as varint - Assert.assertEquals(128, QwpZigZag.encode(64L)); - } - - @Test - public void testSymmetry() { - // Test that encode then decode returns the original value - long[] testValues = { - 0, 1, -1, 2, -2, - 100, -100, - 1000000, -1000000, - Long.MAX_VALUE, Long.MIN_VALUE, - Long.MAX_VALUE / 2, Long.MIN_VALUE / 2 - }; - - for (long value : testValues) { - long encoded = QwpZigZag.encode(value); - long decoded = QwpZigZag.decode(encoded); - Assert.assertEquals("Failed for value: " + value, value, decoded); - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/std/ConcurrentHashMapTest.java b/core/src/test/java/io/questdb/client/test/std/ConcurrentHashMapTest.java deleted file mode 100644 index cebfe86..0000000 --- a/core/src/test/java/io/questdb/client/test/std/ConcurrentHashMapTest.java +++ /dev/null @@ -1,156 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.std; - -import io.questdb.client.std.ConcurrentHashMap; -import org.junit.Test; - -import java.util.Map; - -import static org.junit.Assert.*; - -public class ConcurrentHashMapTest { - - @Test - public void testCaseKey() { - ConcurrentHashMap map = new ConcurrentHashMap<>(4, false); - map.put("Table", "1"); - map.put("tAble", "2"); - map.put("TaBle", "3"); - map.put("TABle", "4"); - map.put("TaBLE", "5"); - map.putIfAbsent("TaBlE", "Hello"); - assertEquals(1, map.size()); - assertEquals(map.get("TABLE"), "5"); - assertEquals(((Map) map).get("TABLE"), "5"); - assertNull(((Map) map).get(42)); - - ConcurrentHashMap cs = new ConcurrentHashMap<>(5, 0.58F); - cs.put("Table", "1"); - cs.put("tAble", "2"); - cs.put("TaBle", "3"); - cs.put("TABle", "4"); - cs.put("TaBLE", "5"); - - ConcurrentHashMap ccs = new ConcurrentHashMap<>(cs); - assertEquals(ccs.size(), cs.size()); - assertEquals(ccs.get("TaBLE"), "5"); - assertNull(ccs.get("TABLE")); - - ConcurrentHashMap cci = new ConcurrentHashMap<>(cs, false); - assertEquals(1, cci.size()); - assertNotNull(cci.get("TaBLE")); - - ConcurrentHashMap ci = new ConcurrentHashMap<>(5, 0.58F, false); - ci.put("Table", "1"); - ci.put("tAble", "2"); - ci.put("TaBle", "3"); - ci.put("TABle", "4"); - ci.put("TaBLE", "5"); - assertEquals(1, ci.size()); - - ConcurrentHashMap.KeySetView ks0 = ConcurrentHashMap.newKeySet(4); - ks0.add("Table"); - ks0.add("tAble"); - ks0.add("TaBle"); - ks0.add("TABle"); - ks0.add("TaBLE"); - assertEquals(5, ks0.size()); - ConcurrentHashMap.KeySetView ks1 = ConcurrentHashMap.newKeySet(4, false); - ks1.add("Table"); - ks1.add("tAble"); - ks1.add("TaBle"); - ks1.add("TABle"); - ks1.add("TaBLE"); - assertEquals(1, ks1.size()); - } - - @Test - public void testCompute() { - ConcurrentHashMap map = identityMap(); - // add - assertEquals("X", map.compute("X", (k, v) -> "X")); - // ignore - map.compute("Y", (k, v) -> null); - assertFalse(map.containsKey("Y")); - // replace - assertEquals("X", map.compute("A", (k, v) -> "X")); - // remove - map.compute("B", (k, v) -> null); - assertFalse(map.containsKey("B")); - - try { - map.compute(null, (k, v) -> null); - fail("Null key"); - } catch (NullPointerException ignored) { - } - } - - @Test - public void testComputeIfAbsent() { - ConcurrentHashMap map = identityMap(); - - map.putIfAbsent("X", "X"); - assertTrue(map.containsKey("X")); - - assertEquals("A", map.computeIfAbsent("A", k -> "X")); - - map.computeIfAbsent("Y", k -> null); - assertFalse(map.containsKey("Y")); - - try { - map.computeIfAbsent(null, k -> null); - fail("Null key"); - } catch (NullPointerException ignored) { - } - } - - @Test - public void testComputeIfPresent() { - ConcurrentHashMap map = identityMap(); - - map.computeIfPresent("X", (k, v) -> "X"); - assertFalse(map.containsKey("X")); - - assertEquals("X", map.computeIfPresent("A", (k, v) -> "X")); - - try { - map.computeIfPresent(null, (k, v) -> null); - fail("Null key"); - } catch (NullPointerException ignored) { - } - } - - private static ConcurrentHashMap identityMap() { - ConcurrentHashMap identity = new ConcurrentHashMap<>(3); - assertTrue(identity.isEmpty()); - identity.put("A", "A"); - identity.put("B", "B"); - identity.put("C", "C"); - assertFalse(identity.isEmpty()); - assertEquals(3, identity.size()); - return identity; - } -} diff --git a/core/src/test/java/io/questdb/client/test/std/ConcurrentIntHashMapTest.java b/core/src/test/java/io/questdb/client/test/std/ConcurrentIntHashMapTest.java deleted file mode 100644 index 8bd759c..0000000 --- a/core/src/test/java/io/questdb/client/test/std/ConcurrentIntHashMapTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.std; - -import io.questdb.client.std.ConcurrentIntHashMap; -import org.junit.Assert; -import org.junit.Test; - -public class ConcurrentIntHashMapTest { - - @Test - public void testCompute() { - ConcurrentIntHashMap map = identityMap(); - // add - Assert.assertEquals(42, (long) map.compute(42, (k, v) -> 42)); - // ignore - map.compute(24, (k, v) -> null); - Assert.assertFalse(map.containsKey(24)); - // replace - Assert.assertEquals(42, (long) map.compute(1, (k, v) -> 42)); - // remove - map.compute(2, (k, v) -> null); - Assert.assertFalse(map.containsKey(2)); - } - - @Test - public void testComputeIfAbsent() { - ConcurrentIntHashMap map = identityMap(); - - map.putIfAbsent(42, 42); - Assert.assertTrue(map.containsKey(42)); - - Assert.assertEquals(1, (long) map.computeIfAbsent(1, k -> 42)); - - map.computeIfAbsent(142, k -> null); - Assert.assertFalse(map.containsKey(142)); - } - - @Test - public void testComputeIfPresent() { - ConcurrentIntHashMap map = identityMap(); - - map.computeIfPresent(42, (k, v) -> 42); - Assert.assertFalse(map.containsKey(42)); - - Assert.assertEquals(42, (long) map.computeIfPresent(1, (k, v) -> 42)); - } - - @Test - public void testNegativeKey() { - ConcurrentIntHashMap map = new ConcurrentIntHashMap<>(); - Assert.assertNull(map.get(-1)); - Assert.assertThrows(IllegalArgumentException.class, () -> map.put(-2, "a")); - Assert.assertThrows(IllegalArgumentException.class, () -> map.putIfAbsent(-3, "b")); - Assert.assertThrows(IllegalArgumentException.class, () -> map.compute(-4, (val1, val2) -> val2)); - Assert.assertThrows(IllegalArgumentException.class, () -> map.computeIfAbsent(-5, (val) -> "c")); - Assert.assertThrows(IllegalArgumentException.class, () -> map.computeIfPresent(-5, (val1, val2) -> val2)); - } - - @Test - public void testSmoke() { - ConcurrentIntHashMap map = new ConcurrentIntHashMap<>(4); - map.put(1, "1"); - map.put(2, "2"); - map.put(3, "3"); - map.put(4, "4"); - map.put(5, "5"); - map.putIfAbsent(5, "Hello"); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(map.get(5), "5"); - Assert.assertNull(map.get(42)); - - ConcurrentIntHashMap.KeySetView ks = ConcurrentIntHashMap.newKeySet(4); - ks.add(1); - ks.add(2); - ks.add(3); - ks.add(4); - ks.add(5); - Assert.assertEquals(5, ks.size()); - } - - private static ConcurrentIntHashMap identityMap() { - ConcurrentIntHashMap identity = new ConcurrentIntHashMap<>(3, 0.9f); - Assert.assertTrue(identity.isEmpty()); - identity.put(1, 1); - identity.put(2, 2); - identity.put(3, 3); - Assert.assertFalse(identity.isEmpty()); - Assert.assertEquals(3, identity.size()); - return identity; - } -} From efa92e3ac7fcb729b248f6eb0ad520b6772905e8 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 12:22:38 +0100 Subject: [PATCH 120/230] Delete QwpBitReader and its test The core module already has a superset QwpBitReader, and QwpGorillaEncoderTest (the only user of the client-side reader) has moved to core. This eliminates the duplicated reader class. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpBitReader.java | 189 ----- .../qwp/protocol/QwpGorillaEncoderTest.java | 715 ------------------ 2 files changed, 904 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java deleted file mode 100644 index a253f6e..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ /dev/null @@ -1,189 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.protocol; - -import io.questdb.client.std.Unsafe; - -/** - * Bit-level reader for ILP v4 protocol. - *

    - * This class reads bits from a buffer in LSB-first order within each byte. - * Bits are read sequentially, spanning byte boundaries as needed. - *

    - * The implementation buffers bytes to minimize memory reads. - *

    - * Usage pattern: - * 

    - * QwpBitReader reader = new QwpBitReader();
- * reader.reset(address, length);
- * int bit = reader.readBit();
- * long value = reader.readBits(numBits);
- * long signedValue = reader.readSigned(numBits);
- *
    - */ -public class QwpBitReader { - - // Buffer for reading bits - private long bitBuffer; - // Number of bits currently available in the buffer (0-64) - private int bitsInBuffer; - private long currentAddress; - private long endAddress; - // Total bits available for reading (from reset) - private long totalBitsAvailable; - // Total bits already consumed - private long totalBitsRead; - - /** - * Creates a new bit reader. Call {@link #reset} before use. - */ - public QwpBitReader() { - } - - /** - * Reads a single bit. - * - * @return 0 or 1 - * @throws IllegalStateException if no more bits available - */ - public int readBit() { - if (totalBitsRead >= totalBitsAvailable) { - throw new IllegalStateException("bit read overflow"); - } - if (!ensureBits(1)) { - throw new IllegalStateException("bit read overflow"); - } - - int bit = (int) (bitBuffer & 1); - bitBuffer >>>= 1; - bitsInBuffer--; - totalBitsRead++; - return bit; - } - - /** - * Reads multiple bits and returns them as a long (unsigned). - *

    - * Bits are returned LSB-aligned. For example, reading 4 bits might return - * 0b1101 where bit 0 is the first bit read. - * - * @param numBits number of bits to read (1-64) - * @return the value formed by the bits (unsigned) - * @throws IllegalStateException if not enough bits available - */ - public long readBits(int numBits) { - if (numBits <= 0) { - return 0; - } - if (numBits > 64) { - throw new IllegalArgumentException("Cannot read more than 64 bits at once"); - } - if (totalBitsRead + numBits > totalBitsAvailable) { - throw new IllegalStateException("bit read overflow"); - } - - long result = 0; - int bitsRemaining = numBits; - int resultShift = 0; - - while (bitsRemaining > 0) { - if (bitsInBuffer == 0) { - if (!ensureBits(Math.min(bitsRemaining, 64))) { - throw new IllegalStateException("bit read overflow"); - } - } - - int bitsToTake = Math.min(bitsRemaining, bitsInBuffer); - long mask = bitsToTake == 64 ? -1L : (1L << bitsToTake) - 1; - result |= (bitBuffer & mask) << resultShift; - - bitBuffer >>>= bitsToTake; - bitsInBuffer -= bitsToTake; - bitsRemaining -= bitsToTake; - resultShift += bitsToTake; - } - - totalBitsRead += numBits; - return result; - } - - /** - * Reads a complete 32-bit integer in little-endian order. - * - * @return the integer value - * @throws IllegalStateException if not enough data - */ - public int readInt() { - return (int) readBits(32); - } - - /** - * Reads multiple bits and interprets them as a signed value using two's complement. - * - * @param numBits number of bits to read (1-64) - * @return the signed value - * @throws IllegalStateException if not enough bits available - */ - public long readSigned(int numBits) { - long unsigned = readBits(numBits); - // Sign extend: if the high bit (bit numBits-1) is set, extend the sign - if (numBits < 64 && (unsigned & (1L << (numBits - 1))) != 0) { - // Set all bits above numBits to 1 - unsigned |= -1L << numBits; - } - return unsigned; - } - - /** - * Resets the reader to read from the specified memory region. - * - * @param address the starting address - * @param length the number of bytes available to read - */ - public void reset(long address, long length) { - this.currentAddress = address; - this.endAddress = address + length; - this.bitBuffer = 0; - this.bitsInBuffer = 0; - this.totalBitsAvailable = length * 8L; - this.totalBitsRead = 0; - } - - /** - * Ensures the buffer has at least the requested number of bits. - * Loads more bytes from memory if needed. - * - * @param bitsNeeded minimum bits required in buffer - * @return true if sufficient bits available, false otherwise - */ - private boolean ensureBits(int bitsNeeded) { - while (bitsInBuffer < bitsNeeded && bitsInBuffer <= 56 && currentAddress < endAddress) { - byte b = Unsafe.getUnsafe().getByte(currentAddress++); - bitBuffer |= (long) (b & 0xFF) << bitsInBuffer; - bitsInBuffer += 8; - } - return bitsInBuffer >= bitsNeeded; - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java deleted file mode 100644 index 09a46dd..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java +++ /dev/null @@ -1,715 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.protocol; - -import io.questdb.client.cutlass.qwp.protocol.QwpBitReader; -import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; -import org.junit.Assert; -import org.junit.Test; - -public class QwpGorillaEncoderTest { - - @Test - public void testCalculateEncodedSizeConstantDelta() throws Exception { - assertMemoryLeak(() -> { - long[] timestamps = new long[100]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = 1_000_000_000L + i * 1000L; - } - long src = putTimestamps(timestamps); - try { - int size = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); - // 8 (first) + 8 (second) + ceil(98 bits / 8) = 29 - int expectedBits = timestamps.length - 2; - int expectedSize = 8 + 8 + (expectedBits + 7) / 8; - Assert.assertEquals(expectedSize, size); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCalculateEncodedSizeEmpty() throws Exception { - assertMemoryLeak(() -> { - Assert.assertEquals(0, QwpGorillaEncoder.calculateEncodedSize(0, 0)); - }); - } - - @Test - public void testCalculateEncodedSizeIdenticalDeltas() throws Exception { - assertMemoryLeak(() -> { - long[] ts = {100L, 200L, 300L}; // delta=100, DoD=0 - long src = putTimestamps(ts); - try { - int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); - // 8 + 8 + 1 byte (1 bit padded to byte) = 17 - Assert.assertEquals(17, size); - } finally { - Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCalculateEncodedSizeOneTimestamp() throws Exception { - assertMemoryLeak(() -> { - long[] ts = {1000L}; - long src = putTimestamps(ts); - try { - Assert.assertEquals(8, QwpGorillaEncoder.calculateEncodedSize(src, 1)); - } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCalculateEncodedSizeSmallDoD() throws Exception { - assertMemoryLeak(() -> { - long[] ts = {100L, 200L, 350L}; // delta0=100, delta1=150, DoD=50 - long src = putTimestamps(ts); - try { - int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); - // 8 + 8 + 2 bytes (9 bits padded to bytes) = 18 - Assert.assertEquals(18, size); - } finally { - Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCalculateEncodedSizeTwoTimestamps() throws Exception { - assertMemoryLeak(() -> { - long[] ts = {1000L, 2000L}; - long src = putTimestamps(ts); - try { - Assert.assertEquals(16, QwpGorillaEncoder.calculateEncodedSize(src, 2)); - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCanUseGorillaConstantDelta() throws Exception { - assertMemoryLeak(() -> { - long[] timestamps = new long[100]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = 1_000_000_000L + i * 1000L; - } - long src = putTimestamps(timestamps); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCanUseGorillaEmpty() throws Exception { - assertMemoryLeak(() -> { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(0, 0)); - }); - } - - @Test - public void testCanUseGorillaLargeDoDOutOfRange() throws Exception { - assertMemoryLeak(() -> { - // DoD = 3_000_000_000 exceeds Integer.MAX_VALUE - long[] timestamps = { - 0L, - 1_000_000_000L, // delta=1_000_000_000 - 5_000_000_000L, // delta=4_000_000_000, DoD=3_000_000_000 - }; - long src = putTimestamps(timestamps); - try { - Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCanUseGorillaNegativeLargeDoDOutOfRange() throws Exception { - assertMemoryLeak(() -> { - // DoD = -4_000_000_000 is less than Integer.MIN_VALUE - long[] timestamps = { - 10_000_000_000L, - 9_000_000_000L, // delta=-1_000_000_000 - 4_000_000_000L, // delta=-5_000_000_000, DoD=-4_000_000_000 - }; - long src = putTimestamps(timestamps); - try { - Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCanUseGorillaOneTimestamp() throws Exception { - assertMemoryLeak(() -> { - long[] ts = {1000L}; - long src = putTimestamps(ts); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 1)); - } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCanUseGorillaTwoTimestamps() throws Exception { - assertMemoryLeak(() -> { - long[] ts = {1000L, 2000L}; - long src = putTimestamps(ts); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 2)); - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCanUseGorillaVaryingDelta() throws Exception { - assertMemoryLeak(() -> { - long[] timestamps = { - 1_000_000_000L, - 1_000_001_000L, // delta=1000 - 1_000_002_100L, // DoD=100 - 1_000_003_500L, // DoD=300 - }; - long src = putTimestamps(timestamps); - try { - Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCompressionRatioConstantInterval() throws Exception { - assertMemoryLeak(() -> { - long[] timestamps = new long[1000]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = i * 1000L; - } - long src = putTimestamps(timestamps); - try { - int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); - int uncompressedSize = timestamps.length * 8; - double ratio = (double) gorillaSize / uncompressedSize; - Assert.assertTrue("Compression ratio should be < 0.1 for constant interval", ratio < 0.1); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testCompressionRatioRandomData() throws Exception { - assertMemoryLeak(() -> { - long[] timestamps = new long[100]; - timestamps[0] = 1_000_000_000L; - timestamps[1] = 1_000_001_000L; - java.util.Random random = new java.util.Random(42); - for (int i = 2; i < timestamps.length; i++) { - timestamps[i] = timestamps[i - 1] + 1000 + random.nextInt(10_000) - 5000; - } - long src = putTimestamps(timestamps); - try { - int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); - Assert.assertTrue("Size should be positive", gorillaSize > 0); - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeDecodeBucketBoundaries() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - // t0=0, t1=10_000, delta0=10_000 - // For DoD=X: delta1 = 10_000+X, so t2 = 20_000+X - long[][] bucketTests = { - {0L, 10_000L, 20_000L}, // DoD = 0 (bucket 0) - {0L, 10_000L, 20_063L}, // DoD = 63 (bucket 1 max) - {0L, 10_000L, 19_936L}, // DoD = -64 (bucket 1 min) - {0L, 10_000L, 20_064L}, // DoD = 64 (bucket 2 start) - {0L, 10_000L, 19_935L}, // DoD = -65 (bucket 2 start) - {0L, 10_000L, 20_255L}, // DoD = 255 (bucket 2 max) - {0L, 10_000L, 19_744L}, // DoD = -256 (bucket 2 min) - {0L, 10_000L, 20_256L}, // DoD = 256 (bucket 3 start) - {0L, 10_000L, 19_743L}, // DoD = -257 (bucket 3 start) - {0L, 10_000L, 22_047L}, // DoD = 2047 (bucket 3 max) - {0L, 10_000L, 17_952L}, // DoD = -2048 (bucket 3 min) - {0L, 10_000L, 22_048L}, // DoD = 2048 (bucket 4 start) - {0L, 10_000L, 17_951L}, // DoD = -2049 (bucket 4 start) - {0L, 10_000L, 110_000L}, // DoD = 100_000 (bucket 4, large) - {0L, 10_000L, -80_000L}, // DoD = -100_000 (bucket 4, large) - }; - - for (long[] tc : bucketTests) { - long src = putTimestamps(tc); - long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, src, tc.length); - Assert.assertTrue("Failed to encode: " + java.util.Arrays.toString(tc), bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(tc[0], first); - Assert.assertEquals(tc[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long dod = decodeDoD(reader); - long delta = (second - first) + dod; - long decoded = second + delta; - Assert.assertEquals("Failed for: " + java.util.Arrays.toString(tc), tc[2], decoded); - } finally { - Unsafe.free(src, (long) tc.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); - } - } - }); - } - - @Test - public void testEncodeTimestampsEmpty() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, 0, 0); - Assert.assertEquals(0, bytesWritten); - } finally { - Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeTimestampsOneTimestamp() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - long[] timestamps = {1_234_567_890L}; - long src = putTimestamps(timestamps); - long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 1); - Assert.assertEquals(8, bytesWritten); - Assert.assertEquals(1_234_567_890L, Unsafe.getUnsafe().getLong(dst)); - } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeTimestampsRoundTripAllBuckets() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - long[] timestamps = new long[10]; - timestamps[0] = 1_000_000_000L; - timestamps[1] = 1_000_001_000L; // delta = 1000 - timestamps[2] = 1_000_002_000L; // DoD=0 (bucket 0) - timestamps[3] = 1_000_003_050L; // DoD=50 (bucket 1) - timestamps[4] = 1_000_003_987L; // DoD=-113 (bucket 2) - timestamps[5] = 1_000_004_687L; // DoD=-237 (bucket 2) - timestamps[6] = 1_000_006_387L; // DoD=1000 (bucket 3) - timestamps[7] = 1_000_020_087L; // DoD=12000 (bucket 4) - timestamps[8] = 1_000_033_787L; // DoD=0 (bucket 0) - timestamps[9] = 1_000_047_487L; // DoD=0 (bucket 0) - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; - } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeTimestampsRoundTripConstantDelta() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - long[] timestamps = new long[100]; - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = 1_000_000_000L + i * 1000L; - } - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; - } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeTimestampsRoundTripLargeDataset() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - int count = 10_000; - long[] timestamps = new long[count]; - timestamps[0] = 1_000_000_000L; - timestamps[1] = 1_000_001_000L; - - java.util.Random random = new java.util.Random(42); - for (int i = 2; i < count; i++) { - long prevDelta = timestamps[i - 1] - timestamps[i - 2]; - int variation = (i % 10 == 0) ? random.nextInt(100) - 50 : 0; - timestamps[i] = timestamps[i - 1] + prevDelta + variation; - } - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, count) + 100; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, count); - Assert.assertTrue(bytesWritten > 0); - Assert.assertTrue("Should compress better than uncompressed", bytesWritten < count * 8); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < count; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; - } - } finally { - Unsafe.free(src, (long) count * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeTimestampsRoundTripNegativeDoD() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - long[] timestamps = { - 1_000_000_000L, - 1_000_002_000L, // delta=2000 - 1_000_003_000L, // DoD=-1000 (bucket 3) - 1_000_003_500L, // DoD=-500 (bucket 2) - 1_000_003_600L, // DoD=-400 (bucket 2) - }; - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; - } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeTimestampsRoundTripVaryingDelta() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - QwpBitReader reader = new QwpBitReader(); - - long[] timestamps = { - 1_000_000_000L, - 1_000_001_000L, // delta=1000 - 1_000_002_000L, // DoD=0 (bucket 0) - 1_000_003_010L, // DoD=10 (bucket 1) - 1_000_004_120L, // DoD=100 (bucket 1) - 1_000_005_420L, // DoD=190 (bucket 2) - 1_000_007_720L, // DoD=1000 (bucket 3) - 1_000_020_020L, // DoD=10000 (bucket 4) - }; - - long src = putTimestamps(timestamps); - int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; - long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); - Assert.assertTrue(bytesWritten > 0); - - long first = Unsafe.getUnsafe().getLong(dst); - long second = Unsafe.getUnsafe().getLong(dst + 8); - Assert.assertEquals(timestamps[0], first); - Assert.assertEquals(timestamps[1], second); - - reader.reset(dst + 16, bytesWritten - 16); - long prevTs = second; - long prevDelta = second - first; - for (int i = 2; i < timestamps.length; i++) { - long dod = decodeDoD(reader); - long delta = prevDelta + dod; - long ts = prevTs + delta; - Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); - prevDelta = delta; - prevTs = ts; - } - } finally { - Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testEncodeTimestampsTwoTimestamps() throws Exception { - assertMemoryLeak(() -> { - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); - long[] timestamps = {1_000_000_000L, 1_000_001_000L}; - long src = putTimestamps(timestamps); - long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); - try { - int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 2); - Assert.assertEquals(16, bytesWritten); - Assert.assertEquals(1_000_000_000L, Unsafe.getUnsafe().getLong(dst)); - Assert.assertEquals(1_000_001_000L, Unsafe.getUnsafe().getLong(dst + 8)); - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); - } - }); - } - - @Test - public void testGetBitsRequired() throws Exception { - assertMemoryLeak(() -> { - // Bucket 0: 1 bit - Assert.assertEquals(1, QwpGorillaEncoder.getBitsRequired(0)); - - // Bucket 1: 9 bits (2 prefix + 7 value) - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(1)); - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-1)); - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(63)); - Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-64)); - - // Bucket 2: 12 bits (3 prefix + 9 value) - Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(64)); - Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(255)); - Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(-256)); - - // Bucket 3: 16 bits (4 prefix + 12 value) - Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(256)); - Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(2047)); - Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(-2048)); - - // Bucket 4: 36 bits (4 prefix + 32 value) - Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(2048)); - Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(-2049)); - }); - } - - @Test - public void testGetBucket12Bit() throws Exception { - assertMemoryLeak(() -> { - // DoD in [-2048, 2047] but outside [-256, 255] -> bucket 3 (16 bits) - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(256)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-257)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(2047)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2047)); - Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2048)); - }); - } - - @Test - public void testGetBucket32Bit() throws Exception { - assertMemoryLeak(() -> { - // DoD outside [-2048, 2047] -> bucket 4 (36 bits) - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(2048)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-2049)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(100_000)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-100_000)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MAX_VALUE)); - Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MIN_VALUE)); - }); - } - - @Test - public void testGetBucket7Bit() throws Exception { - assertMemoryLeak(() -> { - // DoD in [-64, 63] -> bucket 1 (9 bits) - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(1)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-1)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(63)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-63)); - Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-64)); - }); - } - - @Test - public void testGetBucket9Bit() throws Exception { - assertMemoryLeak(() -> { - // DoD in [-256, 255] but outside [-64, 63] -> bucket 2 (12 bits) - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(64)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-65)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(255)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-255)); - Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-256)); - }); - } - - @Test - public void testGetBucketZero() throws Exception { - assertMemoryLeak(() -> { - Assert.assertEquals(0, QwpGorillaEncoder.getBucket(0)); - }); - } - - /** - * Decodes a delta-of-delta value from the bit stream, mirroring the - * core QwpGorillaDecoder.decodeDoD() logic. - */ - private static long decodeDoD(QwpBitReader reader) { - int bit = reader.readBit(); - if (bit == 0) { - return 0; - } - bit = reader.readBit(); - if (bit == 0) { - return reader.readSigned(7); - } - bit = reader.readBit(); - if (bit == 0) { - return reader.readSigned(9); - } - bit = reader.readBit(); - if (bit == 0) { - return reader.readSigned(12); - } - return reader.readSigned(32); - } - - /** - * Writes a Java array of timestamps to off-heap memory. - * - * @param timestamps the timestamps to write - * @return the address of the allocated memory (caller must free) - */ - private static long putTimestamps(long[] timestamps) { - long size = (long) timestamps.length * 8; - long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); - for (int i = 0; i < timestamps.length; i++) { - Unsafe.getUnsafe().putLong(address + (long) i * 8, timestamps[i]); - } - return address; - } -} From 2589cf445ef1e9c1ca3046e25d89007300c0b935 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 12:34:05 +0100 Subject: [PATCH 121/230] Delete unused classes and their tests Remove classes that lost all production callers after the previous round of dead-code removal: - BufferWindowCharSequence (only implementor was GenericLexer) - ImmutableIterator (only implementor was GenericLexer) - CharSequenceHashSet (only used in GenericLexer) - BiIntFunction (only used in ConcurrentIntHashMap) - GeoHashes (only caller was Rnd.nextGeoHash()) Also remove Rnd.nextGeoHash() which has zero callers, and delete the CharSequenceHashSetTest. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cairo/GeoHashes.java | 107 -------------- .../io/questdb/client/std/BiIntFunction.java | 30 ---- .../client/std/BufferWindowCharSequence.java | 29 ---- .../client/std/CharSequenceHashSet.java | 133 ------------------ .../questdb/client/std/ImmutableIterator.java | 38 ----- .../main/java/io/questdb/client/std/Rnd.java | 12 -- .../test/std/CharSequenceHashSetTest.java | 91 ------------ 7 files changed, 440 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cairo/GeoHashes.java delete mode 100644 core/src/main/java/io/questdb/client/std/BiIntFunction.java delete mode 100644 core/src/main/java/io/questdb/client/std/BufferWindowCharSequence.java delete mode 100644 core/src/main/java/io/questdb/client/std/CharSequenceHashSet.java delete mode 100644 core/src/main/java/io/questdb/client/std/ImmutableIterator.java delete mode 100644 core/src/test/java/io/questdb/client/test/std/CharSequenceHashSetTest.java diff --git a/core/src/main/java/io/questdb/client/cairo/GeoHashes.java b/core/src/main/java/io/questdb/client/cairo/GeoHashes.java deleted file mode 100644 index e785f91..0000000 --- a/core/src/main/java/io/questdb/client/cairo/GeoHashes.java +++ /dev/null @@ -1,107 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cairo; - -import io.questdb.client.std.Numbers; -import io.questdb.client.std.NumericException; -import io.questdb.client.std.str.CharSink; - -public class GeoHashes { - - // geohash null value: -1 - // we use the highest bit of every storage size (byte, short, int, long) - // to indicate null value. When a null value is cast down, nullity is - // preserved, i.e. highest bit remains set: - // long nullLong = -1L; - // short nullShort = (short) nullLong; - // nullShort == nullLong; - // in addition, -1 is the first negative non geohash value. - public static final int MAX_STRING_LENGTH = 12; - public static final long NULL = -1L; - - private static final char[] base32 = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' - }; - - public static void append(long hash, int bits, CharSink sink) { - if (hash == GeoHashes.NULL) { - sink.putAscii("null"); - } else { - sink.putAscii('\"'); - if (bits < 0) { - GeoHashes.appendCharsUnsafe(hash, -bits, sink); - } else { - GeoHashes.appendBinaryStringUnsafe(hash, bits, sink); - } - sink.putAscii('\"'); - } - } - - public static void appendBinaryStringUnsafe(long hash, int bits, CharSink sink) { - // Below assertion can happen if there is corrupt metadata - // which should not happen in production code since reader and writer check table metadata - assert bits > 0 && bits <= ColumnType.GEOLONG_MAX_BITS; - for (int i = bits - 1; i >= 0; --i) { - sink.putAscii(((hash >> i) & 1) == 1 ? '1' : '0'); - } - } - - public static void appendChars(long hash, int chars, CharSink sink) { - if (hash != NULL) { - appendCharsUnsafe(hash, chars, sink); - } - } - - public static void appendCharsUnsafe(long hash, int chars, CharSink sink) { - // Below assertion can happen if there is corrupt metadata - // which should not happen in production code since reader and writer check table metadata - assert chars > 0 && chars <= MAX_STRING_LENGTH; - for (int i = chars - 1; i >= 0; --i) { - sink.putAscii(base32[(int) ((hash >> i * 5) & 0x1F)]); - } - } - - public static long fromCoordinatesDeg(double lat, double lon, int bits) throws NumericException { - if (lat < -90.0 || lat > 90.0) { - throw NumericException.instance(); - } - if (lon < -180.0 || lon > 180.0) { - throw NumericException.instance(); - } - if (bits < 0 || bits > ColumnType.GEOLONG_MAX_BITS) { - throw NumericException.instance(); - } - return fromCoordinatesDegUnsafe(lat, lon, bits); - } - - public static long fromCoordinatesDegUnsafe(double lat, double lon, int bits) { - long latq = (long) Math.scalb((lat + 90.0) / 180.0, 32); - long lngq = (long) Math.scalb((lon + 180.0) / 360.0, 32); - return Numbers.interleaveBits(latq, lngq) >>> (64 - bits); - } -} diff --git a/core/src/main/java/io/questdb/client/std/BiIntFunction.java b/core/src/main/java/io/questdb/client/std/BiIntFunction.java deleted file mode 100644 index 5de884f..0000000 --- a/core/src/main/java/io/questdb/client/std/BiIntFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -@FunctionalInterface -public interface BiIntFunction { - R apply(int val1, U val2); -} diff --git a/core/src/main/java/io/questdb/client/std/BufferWindowCharSequence.java b/core/src/main/java/io/questdb/client/std/BufferWindowCharSequence.java deleted file mode 100644 index fdc6ef0..0000000 --- a/core/src/main/java/io/questdb/client/std/BufferWindowCharSequence.java +++ /dev/null @@ -1,29 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -public interface BufferWindowCharSequence extends CharSequence { - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/CharSequenceHashSet.java deleted file mode 100644 index 1788881..0000000 --- a/core/src/main/java/io/questdb/client/std/CharSequenceHashSet.java +++ /dev/null @@ -1,133 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import io.questdb.client.std.str.CharSink; -import io.questdb.client.std.str.Sinkable; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.Arrays; - -public class CharSequenceHashSet extends AbstractCharSequenceHashSet implements Sinkable { - private static final int MIN_INITIAL_CAPACITY = 16; - private final ObjList list; - private boolean hasNull = false; - - public CharSequenceHashSet() { - this(MIN_INITIAL_CAPACITY); - } - - private CharSequenceHashSet(int initialCapacity) { - this(initialCapacity, 0.4); - } - - public CharSequenceHashSet(int initialCapacity, double loadFactor) { - super(initialCapacity, loadFactor); - list = new ObjList<>(free); - clear(); - } - - /** - * Adds key to hash set preserving key uniqueness. - * - * @param key immutable sequence of characters. - * @return false if key is already in the set and true otherwise. - */ - public boolean add(@Nullable CharSequence key) { - if (key == null) { - return addNull(); - } - - int index = keyIndex(key); - if (index < 0) { - return false; - } - - addAt(index, key); - return true; - } - - public void addAt(int index, @NotNull CharSequence key) { - final String s = Chars.toString(key); - keys[index] = s; - list.add(s); - if (--free < 1) { - rehash(); - } - } - - public boolean addNull() { - if (hasNull) { - return false; - } - --free; - hasNull = true; - list.add(null); - return true; - } - - @Override - public final void clear() { - free = capacity; - Arrays.fill(keys, null); - list.clear(); - hasNull = false; - } - - @Override - public boolean contains(@Nullable CharSequence key) { - return key == null ? hasNull : keyIndex(key) < 0; - } - - public CharSequence get(int index) { - return list.getQuick(index); - } - - @Override - public void toSink(@NotNull CharSink sink) { - sink.put(list); - } - - @Override - public String toString() { - return list.toString(); - } - - private void rehash() { - int newCapacity = capacity * 2; - free = capacity = newCapacity; - int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); - this.keys = new CharSequence[len]; - mask = len - 1; - int n = list.size(); - free -= n; - for (int i = 0; i < n; i++) { - final CharSequence key = list.getQuick(i); - keys[keyIndex(key)] = key; - } - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/ImmutableIterator.java b/core/src/main/java/io/questdb/client/std/ImmutableIterator.java deleted file mode 100644 index c974c3e..0000000 --- a/core/src/main/java/io/questdb/client/std/ImmutableIterator.java +++ /dev/null @@ -1,38 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import org.jetbrains.annotations.NotNull; - -import java.util.Iterator; - -public interface ImmutableIterator extends Iterator, Iterable { - - @Override - @NotNull - default Iterator iterator() { - return this; - } -} \ No newline at end of file diff --git a/core/src/main/java/io/questdb/client/std/Rnd.java b/core/src/main/java/io/questdb/client/std/Rnd.java index b7fc787..30d4e67 100644 --- a/core/src/main/java/io/questdb/client/std/Rnd.java +++ b/core/src/main/java/io/questdb/client/std/Rnd.java @@ -24,7 +24,6 @@ package io.questdb.client.std; -import io.questdb.client.cairo.GeoHashes; import io.questdb.client.std.str.StringSink; import io.questdb.client.std.str.Utf16Sink; @@ -187,17 +186,6 @@ public double nextDouble() { return (((long) (nextIntForDouble(26)) << 27) + nextIntForDouble(27)) * DOUBLE_UNIT; } - public long nextGeoHash(int bits) { - double x = nextDouble() * 180.0 - 90.0; - double y = nextDouble() * 360.0 - 180.0; - try { - return GeoHashes.fromCoordinatesDeg(x, y, bits); - } catch (NumericException e) { - // Should never happen - return GeoHashes.NULL; - } - } - public int nextInt(int boundary) { return nextPositiveInt() % boundary; } diff --git a/core/src/test/java/io/questdb/client/test/std/CharSequenceHashSetTest.java b/core/src/test/java/io/questdb/client/test/std/CharSequenceHashSetTest.java deleted file mode 100644 index 246117b..0000000 --- a/core/src/test/java/io/questdb/client/test/std/CharSequenceHashSetTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.std; - -import io.questdb.client.std.CharSequenceHashSet; -import io.questdb.client.std.Rnd; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashSet; - -public class CharSequenceHashSetTest { - - @Test - public void testNullHandling() { - Rnd rnd = new Rnd(); - CharSequenceHashSet set = new CharSequenceHashSet(); - int n = 1000; - - for (int i = 0; i < n; i++) { - set.add(next(rnd).toString()); - } - - Assert.assertFalse(set.contains(null)); - Assert.assertTrue(set.add((CharSequence) null)); - Assert.assertEquals(n + 1, set.size()); - Assert.assertFalse(set.add((CharSequence) null)); - Assert.assertEquals(n + 1, set.size()); - Assert.assertTrue(set.contains(null)); - } - - @Test - public void testStress() { - Rnd rnd = new Rnd(); - CharSequenceHashSet set = new CharSequenceHashSet(); - int n = 10000; - - for (int i = 0; i < n; i++) { - set.add(next(rnd).toString()); - } - - Assert.assertEquals(n, set.size()); - - HashSet check = new HashSet<>(); - for (int i = 0, m = set.size(); i < m; i++) { - check.add(set.get(i).toString()); - } - - Assert.assertEquals(n, check.size()); - - Rnd rnd2 = new Rnd(); - for (int i = 0; i < n; i++) { - Assert.assertTrue("at " + i, set.contains(next(rnd2))); - } - - Assert.assertEquals(n, set.size()); - - Rnd rnd3 = new Rnd(); - for (int i = 0; i < n; i++) { - Assert.assertFalse("at " + i, set.add(next(rnd3))); - } - - Assert.assertEquals(n, set.size()); - } - - private static CharSequence next(Rnd rnd) { - return rnd.nextChars((rnd.nextInt() & 15) + 10); - } -} \ No newline at end of file From ec13a3fd7ab9b17e43e9df87fa159e8ef953e983 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 12:59:49 +0100 Subject: [PATCH 122/230] Delete unused methods and their tests Remove 16 dead public methods/constants that have zero callers in non-test code within the java-questdb-client submodule: Numbers: isPow2(), parseFloat(), parseIPv4Quiet(), parseInt000Greedy(), parseIntQuiet(), parseIntSafely(), parseLong000000Greedy() Unsafe: arrayGetVolatile() (2 overloads), arrayPutOrdered() (2 overloads), defineAnonymousClass(), getNativeAllocator(), setRssMemLimit() Also: Misc.getWorkerAffinity(), Hash.hashLong128_32(), Hash.hashLong128_64(), IOOperation.HEARTBEAT Clean up infrastructure that only served the removed methods: AnonymousClassDefiner interface and its two implementations, NATIVE_ALLOCATORS array and constructNativeAllocator(), INT_OFFSET/INT_SCALE fields, and newly unused imports. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/network/IOOperation.java | 1 - .../main/java/io/questdb/client/std/Hash.java | 11 - .../main/java/io/questdb/client/std/Misc.java | 7 - .../java/io/questdb/client/std/Numbers.java | 160 --------------- .../java/io/questdb/client/std/Unsafe.java | 181 ----------------- .../questdb/client/test/std/NumbersTest.java | 189 ------------------ 6 files changed, 549 deletions(-) diff --git a/core/src/main/java/io/questdb/client/network/IOOperation.java b/core/src/main/java/io/questdb/client/network/IOOperation.java index dca5ac3..b600280 100644 --- a/core/src/main/java/io/questdb/client/network/IOOperation.java +++ b/core/src/main/java/io/questdb/client/network/IOOperation.java @@ -25,7 +25,6 @@ package io.questdb.client.network; public final class IOOperation { - public static final int HEARTBEAT = 8; public static final int READ = 1; public static final int WRITE = 4; diff --git a/core/src/main/java/io/questdb/client/std/Hash.java b/core/src/main/java/io/questdb/client/std/Hash.java index f2a557b..d12b374 100644 --- a/core/src/main/java/io/questdb/client/std/Hash.java +++ b/core/src/main/java/io/questdb/client/std/Hash.java @@ -26,19 +26,8 @@ public final class Hash { - // Constant from Rust compiler's FxHasher. - private static final long M2 = 0x517cc1b727220a95L; - private static final int SPREAD_HASH_BITS = 0x7fffffff; - public static int hashLong128_32(long key1, long key2) { - return (int) hashLong128_64(key1, key2); - } - - public static long hashLong128_64(long key1, long key2) { - return fmix64(key1 * M2 + key2); - } - public static int hashLong32(long k) { return (int) hashLong64(k); } diff --git a/core/src/main/java/io/questdb/client/std/Misc.java b/core/src/main/java/io/questdb/client/std/Misc.java index 2d9e35c..0cae9c9 100644 --- a/core/src/main/java/io/questdb/client/std/Misc.java +++ b/core/src/main/java/io/questdb/client/std/Misc.java @@ -30,7 +30,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.Arrays; public final class Misc { public static final String EOL = "\r\n"; @@ -100,12 +99,6 @@ public static Utf8StringSink getThreadLocalUtf8Sink() { return b; } - public static int[] getWorkerAffinity(int workerCount) { - int[] res = new int[workerCount]; - Arrays.fill(res, -1); - return res; - } - private static void freeObjList0(ObjList list) { for (int i = 0, n = list.size(); i < n; i++) { list.setQuick(i, freeIfCloseable(list.getQuick(i))); diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java index 6156f04..e3d5cbe 100644 --- a/core/src/main/java/io/questdb/client/std/Numbers.java +++ b/core/src/main/java/io/questdb/client/std/Numbers.java @@ -25,7 +25,6 @@ package io.questdb.client.std; import io.questdb.client.std.fastdouble.FastDoubleParser; -import io.questdb.client.std.fastdouble.FastFloatParser; import io.questdb.client.std.str.CharSink; import io.questdb.client.std.str.Utf8Sequence; import jdk.internal.math.FDBigInteger; @@ -490,10 +489,6 @@ public static boolean isNull(float value) { return Float.isNaN(value) || Float.isInfinite(value); } - public static boolean isPow2(int value) { - return value > 0 && (value & (value - 1)) == 0; - } - public static int msb(int value) { return 31 - Integer.numberOfLeadingZeros(value); } @@ -506,10 +501,6 @@ public static double parseDouble(CharSequence sequence) throws NumericException return FastDoubleParser.parseDouble(sequence, true); } - public static float parseFloat(CharSequence sequence) throws NumericException { - return FastFloatParser.parseFloat(sequence, true); - } - public static int parseHexInt(CharSequence sequence) throws NumericException { return parseHexInt(sequence, 0, sequence.length()); } @@ -557,17 +548,6 @@ public static int parseIPv4(CharSequence sequence) throws NumericException { return parseIPv4_0(sequence, 0, sequence.length()); } - public static int parseIPv4Quiet(CharSequence sequence) { - try { - if (sequence == null || Chars.equals("null", sequence)) { - return IPv4_NULL; - } - return parseIPv4(sequence); - } catch (NumericException e) { - return IPv4_NULL; - } - } - public static int parseIPv4_0(CharSequence sequence, final int p, int lim) throws NumericException { if (lim == 0) { throw NumericException.instance().put("empty IPv4 address string"); @@ -657,101 +637,6 @@ public static int parseInt(CharSequence sequence, int p, int lim) throws Numeric return parseInt0(sequence, p, lim); } - public static long parseInt000Greedy(CharSequence sequence, final int p, int lim) throws NumericException { - if (lim == p) { - throw NumericException.instance().put("empty number string"); - } - - boolean negative = sequence.charAt(p) == '-'; - int i = p; - if (negative) { - i++; - } - - if (i >= lim || notDigit(sequence.charAt(i))) { - throw NumericException.instance().put("not a number: ").put(sequence); - } - - int val = 0; - for (; i < lim; i++) { - char c = sequence.charAt(i); - - if (notDigit(c)) { - break; - } - - // val * 10 + (c - '0') - int r = (val << 3) + (val << 1) - (c - '0'); - if (r > val) { - throw NumericException.instance().put("number overflow"); - } - val = r; - } - - final int len = i - p; - - if (len > 3 || val == Integer.MIN_VALUE && !negative) { - throw NumericException.instance().put("number overflow"); - } - - while (i - p < 3) { - val *= 10; - i++; - } - - return encodeLowHighInts(negative ? val : -val, len); - } - - public static int parseIntQuiet(CharSequence sequence) { - try { - if (sequence == null || Chars.equals("NaN", sequence)) { - return Numbers.INT_NULL; - } - return parseInt0(sequence, 0, sequence.length()); - } catch (NumericException e) { - return Numbers.INT_NULL; - } - - } - - public static long parseIntSafely(CharSequence sequence, final int p, int lim) throws NumericException { - if (lim == p) { - throw NumericException.instance().put("empty number string"); - } - - boolean negative = sequence.charAt(p) == '-'; - int i = p; - if (negative) { - i++; - } - - if (i >= lim || notDigit(sequence.charAt(i))) { - throw NumericException.instance().put("not a number: ").put(sequence); - } - - int val = 0; - for (; i < lim; i++) { - char c = sequence.charAt(i); - - if (notDigit(c)) { - break; - } - - // val * 10 + (c - '0') - int r = (val << 3) + (val << 1) - (c - '0'); - if (r > val) { - throw NumericException.instance().put("number overflow"); - } - val = r; - } - - if (val == Integer.MIN_VALUE && !negative) { - throw NumericException.instance().put("number overflow"); - } - - return encodeLowHighInts(negative ? val : -val, i - p); - } - public static long parseLong(CharSequence sequence) throws NumericException { if (sequence == null) { throw NumericException.instance().put("null string"); @@ -766,51 +651,6 @@ public static long parseLong(Utf8Sequence sequence) throws NumericException { return parseLong0(sequence.asAsciiCharSequence(), 0, sequence.size()); } - public static long parseLong000000Greedy(CharSequence sequence, final int p, int lim) throws NumericException { - if (lim == p) { - throw NumericException.instance().put("empty number string"); - } - - boolean negative = sequence.charAt(p) == '-'; - int i = p; - if (negative) { - i++; - } - - if (i >= lim || notDigit(sequence.charAt(i))) { - throw NumericException.instance().put("not a number: ").put(sequence); - } - - int val = 0; - for (; i < lim; i++) { - char c = sequence.charAt(i); - - if (notDigit(c)) { - break; - } - - // val * 10 + (c - '0') - int r = (val << 3) + (val << 1) - (c - '0'); - if (r > val) { - throw NumericException.instance().put("number overflow"); - } - val = r; - } - - final int len = i - p; - - if (len > 6 || val == Integer.MIN_VALUE && !negative) { - throw NumericException.instance().put("number overflow"); - } - - while (i - p < 6) { - val *= 10; - i++; - } - - return encodeLowHighInts(negative ? val : -val, len); - } - public static long spreadBits(long v) { v = (v | (v << 16)) & 0X0000FFFF0000FFFFL; v = (v | (v << 8)) & 0X00FF00FF00FF00FFL; diff --git a/core/src/main/java/io/questdb/client/std/Unsafe.java b/core/src/main/java/io/questdb/client/std/Unsafe.java index 778af39..1e4ad42 100644 --- a/core/src/main/java/io/questdb/client/std/Unsafe.java +++ b/core/src/main/java/io/questdb/client/std/Unsafe.java @@ -26,11 +26,7 @@ // @formatter:off import io.questdb.client.cairo.CairoException; -import org.jetbrains.annotations.Nullable; - -import java.lang.invoke.MethodHandles; import java.lang.reflect.AccessibleObject; -import java.lang.reflect.Array; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.concurrent.atomic.LongAdder; @@ -43,15 +39,12 @@ public final class Unsafe { public static final long BYTE_OFFSET; public static final long BYTE_SCALE; - public static final long INT_OFFSET; - public static final long INT_SCALE; public static final Module JAVA_BASE_MODULE = System.class.getModule(); public static final long LONG_OFFSET; public static final long LONG_SCALE; private static final LongAdder[] COUNTERS = new LongAdder[MemoryTag.SIZE]; private static final long FREE_COUNT_ADDR; private static final long MALLOC_COUNT_ADDR; - private static final long[] NATIVE_ALLOCATORS = new long[MemoryTag.SIZE - NATIVE_DEFAULT]; private static final long[] NATIVE_MEM_COUNTER_ADDRS = new long[MemoryTag.SIZE]; private static final long NON_RSS_MEM_USED_ADDR; private static final long OVERRIDE; @@ -59,7 +52,6 @@ public final class Unsafe { private static final long RSS_MEM_LIMIT_ADDR; private static final long RSS_MEM_USED_ADDR; private static final sun.misc.Unsafe UNSAFE; - private static final AnonymousClassDefiner anonymousClassDefiner; private static final Method implAddExports; private Unsafe() { @@ -73,40 +65,6 @@ public static void addExports(Module from, Module to, String packageName) { } } - public static long arrayGetVolatile(long[] array, int index) { - assert index > -1 && index < array.length; - return Unsafe.getUnsafe().getLongVolatile(array, LONG_OFFSET + ((long) index << LONG_SCALE)); - } - - public static int arrayGetVolatile(int[] array, int index) { - assert index > -1 && index < array.length; - return Unsafe.getUnsafe().getIntVolatile(array, INT_OFFSET + ((long) index << INT_SCALE)); - } - - /** - * This call has Atomic*#lazySet / memory_order_release semantics. - * - * @param array array to put into - * @param index index - * @param value value to put - */ - public static void arrayPutOrdered(long[] array, int index, long value) { - assert index > -1 && index < array.length; - Unsafe.getUnsafe().putOrderedLong(array, LONG_OFFSET + ((long) index << LONG_SCALE), value); - } - - /** - * This call has Atomic*#lazySet / memory_order_release semantics. - * - * @param array array to put into - * @param index index - * @param value value to put - */ - public static void arrayPutOrdered(int[] array, int index, int value) { - assert index > -1 && index < array.length; - Unsafe.getUnsafe().putOrderedInt(array, INT_OFFSET + ((long) index << INT_SCALE), value); - } - public static int byteArrayGetInt(byte[] array, int index) { assert index > -1 && index < array.length - 3; return Unsafe.getUnsafe().getInt(array, BYTE_OFFSET + index); @@ -141,21 +99,6 @@ public static boolean cas(long[] array, int index, long expected, long value) { return Unsafe.cas(array, Unsafe.LONG_OFFSET + (((long) index) << Unsafe.LONG_SCALE), expected, value); } - /** - * Defines a class but does not make it known to the class loader or system dictionary. - *

    - * Equivalent to {@code Unsafe#defineAnonymousClass} and {@code Lookup#defineHiddenClass}, except that - * it does not support constant pool patches. - * - * @param hostClass context for linkage, access control, protection domain, and class loader - * @param data bytes of a class file - * @return Java Class for the given bytecode - */ - @Nullable - public static Class defineAnonymousClass(Class hostClass, byte[] data) { - return anonymousClassDefiner.define(hostClass, data); - } - public static long free(long ptr, long size, int memoryTag) { if (ptr != 0) { Unsafe.getUnsafe().freeMemory(ptr); @@ -199,11 +142,6 @@ public static long getMemUsedByTag(int memoryTag) { return COUNTERS[memoryTag].sum() + UNSAFE.getLongVolatile(null, NATIVE_MEM_COUNTER_ADDRS[memoryTag]); } - /** Returns a `*const QdbAllocator` for use in Rust. */ - public static long getNativeAllocator(int memoryTag) { - return NATIVE_ALLOCATORS[memoryTag - NATIVE_DEFAULT]; - } - public static long getReallocCount() { return UNSAFE.getLongVolatile(null, REALLOC_COUNT_ADDR); } @@ -300,10 +238,6 @@ public static void recordMemAlloc(long size, int memoryTag) { } } - public static void setRssMemLimit(long limit) { - UNSAFE.putLongVolatile(null, RSS_MEM_LIMIT_ADDR, limit); - } - private static long AccessibleObject_override_fieldOffset() { if (isJava8Or11()) { return getFieldOffset(AccessibleObject.class, "override"); @@ -339,19 +273,6 @@ private static void checkAllocLimit(long size, int memoryTag) { } } - /** Allocate a new native allocator object and return its pointer */ - private static long constructNativeAllocator(long nativeMemCountersArray, int memoryTag) { - // See `allocator.rs` for the definition of `QdbAllocator`. - // We construct here via `Unsafe` to avoid having initialization order issues with `Os.java`. - final long allocSize = 8 + 8 + 4; // two longs, one int - final long addr = UNSAFE.allocateMemory(allocSize); - Vect.memset(addr, allocSize, 0); - UNSAFE.putLong(addr, nativeMemCountersArray); - UNSAFE.putLong(addr + 8, NATIVE_MEM_COUNTER_ADDRS[memoryTag]); - UNSAFE.putInt(addr + 16, memoryTag); - return addr; - } - private static boolean getOrdinaryObjectPointersCompressionStatus(boolean is32BitJVM) { class Probe { @SuppressWarnings("unused") @@ -390,92 +311,6 @@ private static int msb(int value) { return 31 - Integer.numberOfLeadingZeros(value); } - interface AnonymousClassDefiner { - Class define(Class hostClass, byte[] data); - } - - /** - * Based on {@code MethodHandles.Lookup#defineHiddenClass}. - */ - static class MethodHandlesClassDefiner implements AnonymousClassDefiner { - private static Method defineMethod; - private static Object hiddenClassOptions; - private static Object lookupBase; - private static long lookupOffset; - - @Nullable - public static MethodHandlesClassDefiner newInstance() { - if (defineMethod == null) { - try { - Field trustedLookupField = MethodHandles.Lookup.class.getDeclaredField("IMPL_LOOKUP"); - lookupBase = UNSAFE.staticFieldBase(trustedLookupField); - lookupOffset = UNSAFE.staticFieldOffset(trustedLookupField); - hiddenClassOptions = hiddenClassOptions("NESTMATE"); - defineMethod = MethodHandles.Lookup.class - .getMethod("defineHiddenClass", byte[].class, boolean.class, hiddenClassOptions.getClass()); - } catch (ReflectiveOperationException e) { - return null; - } - } - return new MethodHandlesClassDefiner(); - } - - @Override - public Class define(Class hostClass, byte[] data) { - try { - MethodHandles.Lookup trustedLookup = (MethodHandles.Lookup) UNSAFE.getObject(lookupBase, lookupOffset); - MethodHandles.Lookup definedLookup = - (MethodHandles.Lookup) defineMethod.invoke(trustedLookup.in(hostClass), data, false, hiddenClassOptions); - return definedLookup.lookupClass(); - } catch (Exception e) { - e.printStackTrace(System.out); - return null; - } - } - - @SuppressWarnings("unchecked") - private static Object hiddenClassOptions(String... options) throws ClassNotFoundException { - @SuppressWarnings("rawtypes") - Class optionClass = Class.forName(MethodHandles.Lookup.class.getName() + "$ClassOption"); - Object classOptions = Array.newInstance(optionClass, options.length); - for (int i = 0; i < options.length; i++) { - Array.set(classOptions, i, Enum.valueOf(optionClass, options[i])); - } - return classOptions; - } - } - - /** - * Based on {@code Unsafe#defineAnonymousClass}. - */ - static class UnsafeClassDefiner implements AnonymousClassDefiner { - - private static Method defineMethod; - - @Nullable - public static UnsafeClassDefiner newInstance() { - if (defineMethod == null) { - try { - defineMethod = sun.misc.Unsafe.class - .getMethod("defineAnonymousClass", Class.class, byte[].class, Object[].class); - } catch (ReflectiveOperationException e) { - return null; - } - } - return new UnsafeClassDefiner(); - } - - @Override - public Class define(Class hostClass, byte[] data) { - try { - return (Class) defineMethod.invoke(UNSAFE, hostClass, data, null); - } catch (Exception e) { - e.printStackTrace(System.out); - return null; - } - } - } - static { try { Field theUnsafe = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -485,23 +320,11 @@ public Class define(Class hostClass, byte[] data) { BYTE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(byte[].class); BYTE_SCALE = msb(Unsafe.getUnsafe().arrayIndexScale(byte[].class)); - INT_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(int[].class); - INT_SCALE = msb(Unsafe.getUnsafe().arrayIndexScale(int[].class)); - LONG_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(long[].class); LONG_SCALE = msb(Unsafe.getUnsafe().arrayIndexScale(long[].class)); OVERRIDE = AccessibleObject_override_fieldOffset(); implAddExports = Module.class.getDeclaredMethod("implAddExports", String.class, Module.class); - - AnonymousClassDefiner classDefiner = UnsafeClassDefiner.newInstance(); - if (classDefiner == null) { - classDefiner = MethodHandlesClassDefiner.newInstance(); - } - if (classDefiner == null) { - throw new InstantiationException("failed to initialize class definer"); - } - anonymousClassDefiner = classDefiner; } catch (ReflectiveOperationException e) { throw new ExceptionInInitializerError(e); } @@ -534,9 +357,5 @@ public Class define(Class hostClass, byte[] data) { NATIVE_MEM_COUNTER_ADDRS[i] = ptr; ptr += 8; } - for (int memoryTag = NATIVE_DEFAULT; memoryTag < MemoryTag.SIZE; ++memoryTag) { - NATIVE_ALLOCATORS[memoryTag - NATIVE_DEFAULT] = constructNativeAllocator( - nativeMemCountersArray, memoryTag); - } } } diff --git a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java index ab2f760..ca6db6b 100644 --- a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java +++ b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java @@ -112,11 +112,6 @@ public void testEmptyDouble() { Numbers.parseDouble("D"); } - @Test(expected = NumericException.class) - public void testEmptyFloat() { - Numbers.parseFloat("f"); - } - @Test(expected = NumericException.class) public void testEmptyLong() { Numbers.parseLong("L"); @@ -317,11 +312,6 @@ public void testHexInt() { public void testIntEdge() { Numbers.append(sink, Integer.MAX_VALUE); assertEquals(Integer.MAX_VALUE, Numbers.parseInt(sink)); - - sink.clear(); - - Numbers.append(sink, Integer.MIN_VALUE); - assertEquals(Integer.MIN_VALUE, Numbers.parseIntQuiet(sink)); } @Test @@ -354,40 +344,6 @@ public void testLongToString() { TestUtils.assertEquals("6103390276", sink); } - @Test(expected = NumericException.class) - public void testParse000Greedy0() throws NumericException { - Numbers.parseInt000Greedy("", 0, 0); - } - - @Test - public void testParse000Greedy1() throws NumericException { - String input = "2"; - long val = Numbers.parseInt000Greedy(input, 0, input.length()); - assertEquals(input.length(), Numbers.decodeHighInt(val)); - assertEquals(200, Numbers.decodeLowInt(val)); - } - - @Test - public void testParse000Greedy2() throws NumericException { - String input = "06"; - long val = Numbers.parseInt000Greedy(input, 0, input.length()); - assertEquals(input.length(), Numbers.decodeHighInt(val)); - assertEquals(60, Numbers.decodeLowInt(val)); - } - - @Test - public void testParse000Greedy3() throws NumericException { - String input = "219"; - long val = Numbers.parseInt000Greedy(input, 0, input.length()); - assertEquals(input.length(), Numbers.decodeHighInt(val)); - assertEquals(219, Numbers.decodeLowInt(val)); - } - - @Test(expected = NumericException.class) - public void testParse000Greedy4() throws NumericException { - Numbers.parseInt000Greedy("1234", 0, 4); - } - @Test public void testParseDouble() { @@ -496,123 +452,6 @@ public void testParseExplicitDouble() { assertEquals(1234.123d, Numbers.parseDouble("1234.123d"), 0.000001); } - @Test - public void testParseExplicitFloat() { - assertEquals(12345.02f, Numbers.parseFloat("12345.02f"), 0.0001f); - } - - @Test(expected = NumericException.class) - public void testParseExplicitFloat2() { - Numbers.parseFloat("12345.02fx"); - } - - @Test - public void testParseFloat() { - String s1 = "0.45677899234"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "1.459983E35"; - assertEquals(Float.parseFloat(s2) / 1e35d, Numbers.parseFloat(s2) / 1e35d, 0.00001); - - String s3 = "0.000000023E-30"; - assertEquals(Float.parseFloat(s3), Numbers.parseFloat(s3), 0.000000001); - - // overflow - try { - Numbers.parseFloat("1.0000E-204"); - Assert.fail(); - } catch (NumericException ignored) { - } - - try { - Numbers.parseFloat("1E39"); - Assert.fail(); - } catch (NumericException ignored) { - } - - try { - Numbers.parseFloat("1.0E39"); - Assert.fail(); - } catch (NumericException ignored) { - } - - String s6 = "200E2"; - assertEquals(Float.parseFloat(s6), Numbers.parseFloat(s6), 0.000000001); - - String s7 = "NaN"; - assertEquals(Float.parseFloat(s7), Numbers.parseFloat(s7), 0.000000001); - - String s8 = "-Infinity"; - assertEquals(Float.parseFloat(s8), Numbers.parseFloat(s8), 0.000000001); - - // min exponent float - String s9 = "1.4e-45"; - assertEquals(1.4e-45f, Numbers.parseFloat(s9), 0.001); - - // false overflow - String s10 = "0003000.0e-46"; - assertEquals(1.4e-45f, Numbers.parseFloat(s10), 0.001); - - // false overflow - String s11 = "0.00001e40"; - assertEquals(1e35f, Numbers.parseFloat(s11), 0.001); - } - - @Test - public void testParseFloatCloseToZero() { - String s1 = "0.123456789"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "0.12345678901234567890123456789E12"; - assertEquals(Float.parseFloat(s2), Numbers.parseFloat(s2), 0.000000001); - } - - @Test - public void testParseFloatIntegerLargerThanLongMaxValue() { - String s1 = "9223372036854775808"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "9223372036854775808123"; - assertEquals(Float.parseFloat(s2), Numbers.parseFloat(s2), 0.000000001); - - String s3 = "9223372036854775808123922337203685477"; - assertEquals(Float.parseFloat(s3), Numbers.parseFloat(s3), 0.000000001); - - String s4 = "92233720368547758081239223372036854771"; - assertEquals(Float.parseFloat(s4), Numbers.parseFloat(s4), 0.000000001); - } - - @Test - public void testParseFloatLargerThanLongMaxValue() throws NumericException { - String s1 = "9223372036854775808.0123456789"; - assertEquals(Float.parseFloat(s1), Numbers.parseFloat(s1), 0.000000001); - - String s2 = "9223372036854775808.0123456789"; - assertEquals(Float.parseFloat(s2), Numbers.parseFloat(s2), 0.000000001); - - String s3 = "9223372036854775808123.0123456789"; - assertEquals(Float.parseFloat(s3), Numbers.parseFloat(s3), 0.000000001); - - String s4 = "922337203685477580812392233720368547758081.01239223372036854775808123"; // overflow - try { - Numbers.parseFloat(s4); - Assert.fail(); - } catch (NumericException ignored) { - } - } - - @Test - public void testParseFloatNegativeZero() throws NumericException { - float actual = Numbers.parseFloat("-0.0"); - - //check it's zero at all - assertEquals(0, actual, 0.0); - - //check it's *negative* zero - float res = 1 / actual; - assertEquals(Float.NEGATIVE_INFINITY, res, 0.0); - } - @Test public void testParseIPv4() { assertEquals(84413540, Numbers.parseIPv4("5.8.12.100")); @@ -677,12 +516,6 @@ public void testParseIPv4Overflow3() { Numbers.parseIPv4("12.1.3500.2"); } - @Test - public void testParseIPv4Quiet() { - assertEquals(0, Numbers.parseIPv4Quiet(null)); - assertEquals(0, Numbers.parseIPv4Quiet("NaN")); - } - @Test(expected = NumericException.class) public void testParseIPv4SignOnly() { Numbers.parseIPv4("-"); @@ -793,28 +626,6 @@ public void testParseIntSignOnly() { Numbers.parseInt("-"); } - @Test - public void testParseIntToDelim() { - String in = "1234x5"; - long val = Numbers.parseIntSafely(in, 0, in.length()); - assertEquals(1234, Numbers.decodeLowInt(val)); - assertEquals(4, Numbers.decodeHighInt(val)); - } - - @Test(expected = NumericException.class) - public void testParseIntToDelimEmpty() { - String in = "x"; - Numbers.parseIntSafely(in, 0, in.length()); - } - - @Test - public void testParseIntToDelimNoChar() { - String in = "12345"; - long val = Numbers.parseIntSafely(in, 0, in.length()); - assertEquals(12345, Numbers.decodeLowInt(val)); - assertEquals(5, Numbers.decodeHighInt(val)); - } - @Test(expected = NumericException.class) public void testParseIntWrongChars() { Numbers.parseInt("123ab"); From 8e77d28b6645ec44819038ccfc3c2eb4571d7ece Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 13:33:52 +0100 Subject: [PATCH 123/230] Delete unused methods and their tests Remove dead methods with no production callers from five classes: Numbers.java: appendLong256(), appendUuid(), appendHexPadded(long, int), intToIPv4Sink(), and their private helpers (appendLong256Four/Three/Two). Chars.java: contains(), noMatch(). Utf8s.java: stringFromUtf8BytesSafe(), both toString() overloads, validateUtf8() and its four private helpers. ColumnType.java: encodeArrayTypeWithWeakDims(). Update test code that referenced deleted methods: TestUtils replaces Chars.contains() with String.contains() and inlines ipv4ToString(). TestHttpClient replaces Utf8s.toString() with a direct null-safe call. ColumnTypeTest and NumbersTest drop tests for removed methods. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cairo/ColumnType.java | 13 --- .../java/io/questdb/client/std/Chars.java | 16 --- .../java/io/questdb/client/std/Numbers.java | 95 ---------------- .../java/io/questdb/client/std/str/Utf8s.java | 105 ------------------ .../client/test/cairo/ColumnTypeTest.java | 10 +- .../test/cutlass/http/TestHttpClient.java | 4 +- .../questdb/client/test/std/NumbersTest.java | 38 ------- .../client/test/std/str/Utf8sTest.java | 9 -- .../questdb/client/test/tools/TestUtils.java | 10 +- 9 files changed, 7 insertions(+), 293 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cairo/ColumnType.java b/core/src/main/java/io/questdb/client/cairo/ColumnType.java index b00046f..275650b 100644 --- a/core/src/main/java/io/questdb/client/cairo/ColumnType.java +++ b/core/src/main/java/io/questdb/client/cairo/ColumnType.java @@ -214,19 +214,6 @@ public static int encodeArrayType(int elemType, int nDims, boolean checkSupporte | ARRAY; } - /** - * Encodes an array type with weak dimensionality. The dimensionality is still - * encoded but marked as tentative and can be updated based on actual data. - * This is useful for PostgreSQL wire protocol where type information doesn't - * include array dimensions. - *

    - * The number of dimensions of this type is undefined, so the decoded number on - * dimensions for the returned column type will be -1. - */ - public static int encodeArrayTypeWithWeakDims(short elemType, boolean checkSupportedElementTypes) { - return encodeArrayType(elemType, 1, checkSupportedElementTypes) | TYPE_FLAG_ARRAY_WEAK_DIMS; - } - /** * Generate a decimal type from a given precision and scale. * It will choose the proper subtype (DECIMAL8, DECIMAL16, etc.) from the precision, depending on the amount diff --git a/core/src/main/java/io/questdb/client/std/Chars.java b/core/src/main/java/io/questdb/client/std/Chars.java index 84985c5..91bf7d7 100644 --- a/core/src/main/java/io/questdb/client/std/Chars.java +++ b/core/src/main/java/io/questdb/client/std/Chars.java @@ -51,10 +51,6 @@ public static void base64Encode(@Nullable BinarySequence sequence, int maxLength } } - public static boolean contains(@NotNull CharSequence sequence, @NotNull CharSequence term) { - return indexOf(sequence, 0, sequence.length(), term) != -1; - } - public static boolean equals(@NotNull CharSequence l, @NotNull CharSequence r) { if (l == r) { return true; @@ -356,18 +352,6 @@ public static int lowerCaseHashCode(CharSequence value) { return h; } - public static boolean noMatch(CharSequence l, int llo, int lhi, CharSequence r, int rlo, int rhi) { - int lp = llo; - int rp = rlo; - while (lp < lhi && rp < rhi) { - if (Character.toLowerCase(l.charAt(lp++)) != r.charAt(rp++)) { - return true; - } - - } - return lp != lhi || rp != rhi; - } - public static boolean startsWith(@Nullable CharSequence cs, @Nullable CharSequence starts) { if (cs == null || starts == null) { return false; diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java index e3d5cbe..192f63c 100644 --- a/core/src/main/java/io/questdb/client/std/Numbers.java +++ b/core/src/main/java/io/questdb/client/std/Numbers.java @@ -262,43 +262,6 @@ public static void appendHex(CharSink sink, long value, boolean pad) { array[bit].append(sink, value); } - /** - * Append a long value to a CharSink in hex format. - * - * @param sink the CharSink to append to - * @param value the value to append - * @param padToBytes if non-zero, pad the output to the specified number of bytes - */ - public static void appendHexPadded(CharSink sink, long value, int padToBytes) { - assert padToBytes >= 0 && padToBytes <= 8; - // This code might be unclear, so here are some hints: - // This method uses longHexAppender() and longHexAppender() is always padding to a whole byte. It never prints - // just a nibble. It means the longHexAppender() will print value 0xf as "0f". Value 0xff will be printed as "ff". - // Value 0xfff will be printed as "0fff". Value 0xffff will be printed as "ffff" and so on. - // So this method needs to pad only from the next whole byte up. - // In other words: This method always pads with full bytes (=even number of zeros), never with just a nibble. - - // Example 1: Value is 0xF and padToBytes is 2. This means the desired output is 000f. - // longHexAppender() pads to a full byte. This means it will output is 0f. So this method needs to pad with 2 zeros. - - // Example 2: The value is 0xFF and padToBytes is 2. This means the desired output is 00ff. - // longHexAppender() will output "ff". This is a full byte so longHexAppender() will not do any padding on its own. - // So this method needs to pad with 2 zeros. - int leadingZeroBits = Long.numberOfLeadingZeros(value); - int padToBits = padToBytes << 3; - int bitsToPad = padToBits - (Long.SIZE - leadingZeroBits); - int bytesToPad = (bitsToPad >> 3); - for (int i = 0; i < bytesToPad; i++) { - sink.putAscii('0'); - sink.putAscii('0'); - } - if (value == 0) { - return; - } - int bit = 64 - leadingZeroBits; - longHexAppender[bit].append(sink, value); - } - public static void appendHexPadded(CharSink sink, final int value) { int i = value; if (i < 0) { @@ -385,38 +348,6 @@ public static void appendHexPadded(CharSink sink, final int value) { } } - public static void appendLong256(long a, long b, long c, long d, CharSink sink) { - if (a == Numbers.LONG_NULL && b == Numbers.LONG_NULL && c == Numbers.LONG_NULL && d == Numbers.LONG_NULL) { - return; - } - sink.putAscii("0x"); - if (d != 0) { - appendLong256Four(a, b, c, d, sink); - return; - } - if (c != 0) { - appendLong256Three(a, b, c, sink); - return; - } - if (b != 0) { - appendLong256Two(a, b, sink); - return; - } - appendHex(sink, a, false); - } - - public static void appendUuid(long lo, long hi, CharSink sink) { - appendHexPadded(sink, (hi >> 32) & 0xFFFFFFFFL, 4); - sink.putAscii('-'); - appendHexPadded(sink, (hi >> 16) & 0xFFFF, 2); - sink.putAscii('-'); - appendHexPadded(sink, hi & 0xFFFF, 2); - sink.putAscii('-'); - appendHexPadded(sink, lo >> 48 & 0xFFFF, 2); - sink.putAscii('-'); - appendHexPadded(sink, lo & 0xFFFFFFFFFFFFL, 6); - } - public static int ceilPow2(int value) { int i = value; if ((i != 0) && (i & (i - 1)) > 0) { @@ -458,17 +389,6 @@ public static int hexToDecimal(int c) throws NumericException { return r; } - public static void intToIPv4Sink(CharSink sink, int value) { - // NULL handling should be done outside, null here will be printed as 0.0.0.0 - append(sink, (value >> 24) & 0xff); - sink.putAscii('.'); - append(sink, (value >> 16) & 0xff); - sink.putAscii('.'); - append(sink, (value >> 8) & 0xff); - sink.putAscii('.'); - append(sink, value & 0xff); - } - public static long interleaveBits(long x, long y) { return spreadBits(x) | (spreadBits(y) << 1); } @@ -1280,21 +1200,6 @@ private static void appendLong2(CharSink sink, long i) { sink.putAscii((char) ('0' + i % 10)); } - private static void appendLong256Four(long a, long b, long c, long d, CharSink sink) { - appendLong256Three(b, c, d, sink); - appendHex(sink, a, true); - } - - private static void appendLong256Three(long a, long b, long c, CharSink sink) { - appendLong256Two(b, c, sink); - appendHex(sink, a, true); - } - - private static void appendLong256Two(long a, long b, CharSink sink) { - appendHex(sink, b, false); - appendHex(sink, a, true); - } - private static void appendLong3(CharSink sink, long i) { long c; sink.putAscii((char) ('0' + i / 100)); diff --git a/core/src/main/java/io/questdb/client/std/str/Utf8s.java b/core/src/main/java/io/questdb/client/std/str/Utf8s.java index e1986cd..4db4641 100644 --- a/core/src/main/java/io/questdb/client/std/str/Utf8s.java +++ b/core/src/main/java/io/questdb/client/std/str/Utf8s.java @@ -225,32 +225,6 @@ public static String stringFromUtf8Bytes(@NotNull Utf8Sequence seq) { return b.toString(); } - public static String stringFromUtf8BytesSafe(@NotNull Utf8Sequence seq) { - if (seq.size() == 0) { - return ""; - } - Utf16Sink b = getThreadLocalSink(); - utf8ToUtf16(seq, b); - return b.toString(); - } - - public static String toString(@Nullable Utf8Sequence s) { - return s == null ? null : s.toString(); - } - - public static String toString(@NotNull Utf8Sequence us, int start, int end, byte unescapeAscii) { - final Utf8Sink sink = getThreadLocalUtf8Sink(); - final int lastChar = end - 1; - for (int i = start; i < end; i++) { - byte b = us.byteAt(i); - sink.putAny(b); - if (b == unescapeAscii && i < lastChar && us.byteAt(i + 1) == unescapeAscii) { - i++; - } - } - return sink.toString(); - } - public static int utf8DecodeMultiByte(long lo, long hi, byte b, Utf16Sink sink) { if (b >> 5 == -2 && (b & 30) != 0) { return utf8Decode2Bytes(lo, hi, b, sink); @@ -329,28 +303,6 @@ public static boolean utf8ToUtf16(@NotNull Utf8Sequence seq, @NotNull Utf16Sink return utf8ToUtf16(seq, 0, seq.size(), sink); } - public static int validateUtf8(@NotNull Utf8Sequence seq) { - if (seq.isAscii()) { - return seq.size(); - } - int len = 0; - for (int i = 0, hi = seq.size(); i < hi; ) { - byte b = seq.byteAt(i); - if (b < 0) { - int n = validateUtf8MultiByte(seq, i, b); - if (n == -1) { - // UTF-8 error - return -1; - } - i += n; - } else { - ++i; - } - ++len; - } - return len; - } - /** * Returns up to 6 initial bytes of the given UTF-8 sequence (less if it's shorter) * packed into a zero-padded long value, in little-endian order. This prefix is @@ -674,61 +626,4 @@ private static int utf8DecodeMultiByte(Utf8Sequence seq, int index, byte b, @Not return utf8Decode4Bytes(seq, index, b, sink); } - private static int validateUtf8Decode2Bytes(@NotNull Utf8Sequence seq, int index) { - if (seq.size() - index < 2) { - return -1; - } - byte b2 = seq.byteAt(index + 1); - if (isNotContinuation(b2)) { - return -1; - } - return 2; - } - - private static int validateUtf8Decode3Bytes(@NotNull Utf8Sequence seq, int index, byte b1) { - if (seq.size() - index < 3) { - return -1; - } - byte b2 = seq.byteAt(index + 1); - byte b3 = seq.byteAt(index + 2); - - if (isMalformed3(b1, b2, b3)) { - return -1; - } - - char c = utf8ToChar(b1, b2, b3); - if (Character.isSurrogate(c)) { - return -1; - } - return 3; - } - - private static int validateUtf8Decode4Bytes(@NotNull Utf8Sequence seq, int index, int b) { - if (b >> 3 != -2 || seq.size() - index < 4) { - return -1; - } - byte b2 = seq.byteAt(index + 1); - byte b3 = seq.byteAt(index + 2); - byte b4 = seq.byteAt(index + 3); - - if (isMalformed4(b2, b3, b4)) { - return -1; - } - final int codePoint = getUtf8Codepoint(b, b2, b3, b4); - if (!Character.isSupplementaryCodePoint(codePoint)) { - return -1; - } - return 4; - } - - private static int validateUtf8MultiByte(Utf8Sequence seq, int index, byte b) { - if (b >> 5 == -2 && (b & 30) != 0) { - // we should allow 11000001, as it is a valid UTF8 byte? - return validateUtf8Decode2Bytes(seq, index); - } - if (b >> 4 == -2) { - return validateUtf8Decode3Bytes(seq, index, b); - } - return validateUtf8Decode4Bytes(seq, index, b); - } } \ No newline at end of file diff --git a/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java b/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java index debb3ed..20429a0 100644 --- a/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java +++ b/core/src/test/java/io/questdb/client/test/cairo/ColumnTypeTest.java @@ -30,14 +30,8 @@ public class ColumnTypeTest { @Test - public void testArrayWithWeakDims() { - int arrayType = ColumnType.encodeArrayTypeWithWeakDims(ColumnType.DOUBLE, true); - Assert.assertTrue(ColumnType.isArray(arrayType)); - // arrays with weak dimensions are considered undefined - Assert.assertEquals(ColumnType.DOUBLE, ColumnType.decodeArrayElementType(arrayType)); - Assert.assertEquals(-1, ColumnType.decodeWeakArrayDimensionality(arrayType)); - - arrayType = ColumnType.encodeArrayType(ColumnType.DOUBLE, 5); + public void testArrayEncoding() { + int arrayType = ColumnType.encodeArrayType(ColumnType.DOUBLE, 5); Assert.assertTrue(ColumnType.isArray(arrayType)); Assert.assertEquals(ColumnType.DOUBLE, ColumnType.decodeArrayElementType(arrayType)); Assert.assertEquals(5, ColumnType.decodeWeakArrayDimensionality(arrayType)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java b/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java index af938b3..67bba2b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/TestHttpClient.java @@ -32,7 +32,6 @@ import io.questdb.client.std.str.MutableUtf8Sink; import io.questdb.client.std.str.Utf8Sequence; import io.questdb.client.std.str.Utf8StringSink; -import io.questdb.client.std.str.Utf8s; import io.questdb.client.test.tools.TestUtils; import org.jetbrains.annotations.Nullable; import org.junit.Assert; @@ -489,7 +488,8 @@ protected String reqToSink0( @SuppressWarnings("resource") HttpClient.ResponseHeaders rsp = req.send(); rsp.await(); - String statusCode = Utf8s.toString(rsp.getStatusCode()); + Utf8Sequence sc = rsp.getStatusCode(); + String statusCode = sc == null ? null : sc.toString(); sink.clear(); rsp.getResponse().copyTextTo(sink); return statusCode; diff --git a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java index ca6db6b..3554c71 100644 --- a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java +++ b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java @@ -41,37 +41,6 @@ public class NumbersTest { private final StringSink sink = new StringSink(); private Rnd rnd; - @Test - public void appendHexPadded() { - sink.clear(); - Numbers.appendHexPadded(sink, 0xff - 1, 2); - TestUtils.assertEquals("00fe", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xff0, 4); - TestUtils.assertEquals("00000ff0", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 1, 4); - TestUtils.assertEquals("00000001", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xff - 1, 3); - TestUtils.assertEquals("0000fe", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xff - 1, 1); - TestUtils.assertEquals("fe", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0xffff, 0); - TestUtils.assertEquals("ffff", sink); - - sink.clear(); - Numbers.appendHexPadded(sink, 0, 8); - TestUtils.assertEquals("0000000000000000", sink); - } - @Test(expected = NumericException.class) public void parseExplicitDouble2() { Numbers.parseDouble("1234dx"); @@ -93,13 +62,6 @@ public void setUp() { sink.clear(); } - @Test - public void testAppendZeroLong256() { - sink.clear(); - Numbers.appendLong256(0, 0, 0, 0, sink); - TestUtils.assertEquals("0x00", sink); - } - @Test public void testCeilPow2() { assertEquals(16, ceilPow2(15)); diff --git a/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java b/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java index 8201911..e79d3e6 100644 --- a/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java +++ b/core/src/test/java/io/questdb/client/test/std/str/Utf8sTest.java @@ -272,15 +272,6 @@ public void testUtf8Support() { } } - @Test - public void testValidateUtf8() { - Assert.assertEquals(0, Utf8s.validateUtf8(Utf8String.EMPTY)); - Assert.assertEquals(3, Utf8s.validateUtf8(utf8("abc"))); - Assert.assertEquals(10, Utf8s.validateUtf8(utf8("привет мир"))); - // invalid UTF-8 - Assert.assertEquals(-1, Utf8s.validateUtf8(new Utf8String(new byte[]{(byte) 0x80}, false))); - } - private static byte b(int n) { return (byte) n; } diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java index 532193c..125f728 100644 --- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java @@ -25,12 +25,10 @@ package io.questdb.client.test.tools; import io.questdb.client.std.BinarySequence; -import io.questdb.client.std.Chars; import io.questdb.client.std.Files; import io.questdb.client.std.IntList; import io.questdb.client.std.LongList; import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Numbers; import io.questdb.client.std.ObjList; import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; @@ -86,7 +84,7 @@ public static void assertContains(String message, CharSequence sequence, CharSeq if (term.length() == 0) { return; } - if (Chars.contains(sequence, term)) { + if (sequence.toString().contains(term.toString())) { return; } Assert.fail((message != null ? message + ": '" : "'") + sequence + "' does not contain: " + term); @@ -351,7 +349,7 @@ public static void assertNotContains(String message, CharSequence sequence, Char Assert.fail(formatted + "Cannot assert that sequence does not contain an empty term; an empty term is always considered contained by definition."); } - if (!Chars.contains(sequence, term)) { + if (!sequence.toString().contains(term.toString())) { return; } @@ -448,9 +446,7 @@ public static String getTestResourcePath(String resourceName) { } public static String ipv4ToString(int ip) { - StringSink sink = getTlSink(); - Numbers.intToIPv4Sink(sink, ip); - return sink.toString(); + return ((ip >> 24) & 0xff) + "." + ((ip >> 16) & 0xff) + "." + ((ip >> 8) & 0xff) + "." + (ip & 0xff); } public static String readStringFromFile(File file) { From 8d61c15417a6205c100592422f61b56ee2e389e5 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 13:47:45 +0100 Subject: [PATCH 124/230] Remove unused RSS memory limit mechanism The client's Unsafe class had getRssMemLimit(), checkAllocLimit(), and RSS_MEM_LIMIT_ADDR, but no setRssMemLimit() method. The limit address was always zero, making the allocation limit check in malloc() and realloc() dead code. Remove the field, both methods, the static initializer slot, and the now-stale allocator.rs layout comment. Co-Authored-By: Claude Opus 4.6 --- .../java/io/questdb/client/std/Unsafe.java | 37 +------------------ 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/io/questdb/client/std/Unsafe.java b/core/src/main/java/io/questdb/client/std/Unsafe.java index 1e4ad42..6395d99 100644 --- a/core/src/main/java/io/questdb/client/std/Unsafe.java +++ b/core/src/main/java/io/questdb/client/std/Unsafe.java @@ -24,15 +24,13 @@ package io.questdb.client.std; -// @formatter:off import io.questdb.client.cairo.CairoException; + import java.lang.reflect.AccessibleObject; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.concurrent.atomic.LongAdder; -import static io.questdb.client.std.MemoryTag.NATIVE_DEFAULT; - public final class Unsafe { // The various _ADDR fields are `long` in Java, but they are `* mut usize` in Rust, or `size_t*` in C. // These are off-heap allocated atomic counters for memory usage tracking. @@ -49,7 +47,6 @@ public final class Unsafe { private static final long NON_RSS_MEM_USED_ADDR; private static final long OVERRIDE; private static final long REALLOC_COUNT_ADDR; - private static final long RSS_MEM_LIMIT_ADDR; private static final long RSS_MEM_USED_ADDR; private static final sun.misc.Unsafe UNSAFE; private static final Method implAddExports; @@ -146,10 +143,6 @@ public static long getReallocCount() { return UNSAFE.getLongVolatile(null, REALLOC_COUNT_ADDR); } - public static long getRssMemLimit() { - return UNSAFE.getLongVolatile(null, RSS_MEM_LIMIT_ADDR); - } - public static long getRssMemUsed() { return UNSAFE.getLongVolatile(null, RSS_MEM_USED_ADDR); } @@ -183,7 +176,6 @@ public static void makeAccessible(AccessibleObject accessibleObject) { public static long malloc(long size, int memoryTag) { try { assert memoryTag >= MemoryTag.NATIVE_PATH; - checkAllocLimit(size, memoryTag); long ptr = Unsafe.getUnsafe().allocateMemory(size); recordMemAlloc(size, memoryTag); incrMallocCount(); @@ -205,7 +197,6 @@ public static long malloc(long size, int memoryTag) { public static long realloc(long address, long oldSize, long newSize, int memoryTag) { try { assert memoryTag >= MemoryTag.NATIVE_PATH; - checkAllocLimit(-oldSize + newSize, memoryTag); long ptr = Unsafe.getUnsafe().reallocateMemory(address, newSize); recordMemAlloc(-oldSize + newSize, memoryTag); incrReallocCount(); @@ -253,26 +244,6 @@ private static long AccessibleObject_override_fieldOffset() { return 16L; } - private static void checkAllocLimit(long size, int memoryTag) { - if (size <= 0) { - return; - } - // Don't check limits for mmap'd memory - final long rssMemLimit = getRssMemLimit(); - if (rssMemLimit > 0 && memoryTag >= NATIVE_DEFAULT) { - long usage = getRssMemUsed(); - if (usage + size > rssMemLimit) { - throw CairoException.nonCritical() - .put("global RSS memory limit exceeded [usage=") - .put(usage) - .put(", RSS_MEM_LIMIT=").put(rssMemLimit) - .put(", size=").put(size) - .put(", memoryTag=").put(memoryTag) - .put(']'); - } - } - } - private static boolean getOrdinaryObjectPointersCompressionStatus(boolean is32BitJVM) { class Probe { @SuppressWarnings("unused") @@ -333,17 +304,13 @@ private static int msb(int value) { // A single allocation for all the off-heap native memory counters. // Might help with locality, given they're often incremented together. // All initial values set to 0. - final long nativeMemCountersArraySize = (6 + COUNTERS.length) * 8; + final long nativeMemCountersArraySize = (5 + COUNTERS.length) * 8; final long nativeMemCountersArray = UNSAFE.allocateMemory(nativeMemCountersArraySize); long ptr = nativeMemCountersArray; Vect.memset(nativeMemCountersArray, nativeMemCountersArraySize, 0); - // N.B.: The layout here is also used in `allocator.rs` for the Rust side. - // See: `struct MemTracking`. RSS_MEM_USED_ADDR = ptr; ptr += 8; - RSS_MEM_LIMIT_ADDR = ptr; - ptr += 8; MALLOC_COUNT_ADDR = ptr; ptr += 8; REALLOC_COUNT_ADDR = ptr; From 80f720c3f8d59752f8c774957159b50ce17ebb76 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 13:56:54 +0100 Subject: [PATCH 125/230] Delete dead external-buffer mechanism QwpWebSocketEncoder had two buffer fields: ownedBuffer (the allocated NativeBufferWriter) and buffer (the active write target, typed as QwpBufferWriter interface). The indirection existed so callers could inject an external buffer via setBuffer(), but no code ever called that method. The split caused a use-after-free: close() freed and nulled ownedBuffer but left buffer pointing to the closed writer. Merge the two fields into a single NativeBufferWriter buffer. Delete setBuffer(), isUsingExternalBuffer(), and the conditional guard in reset(). close() now nulls the only reference. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketEncoder.java | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 8ae2c14..9a5f7ea 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -51,27 +51,24 @@ public class QwpWebSocketEncoder implements QuietCloseable { public static final byte ENCODING_GORILLA = 0x01; public static final byte ENCODING_UNCOMPRESSED = 0x00; private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); - private QwpBufferWriter buffer; + private NativeBufferWriter buffer; private byte flags; - private NativeBufferWriter ownedBuffer; public QwpWebSocketEncoder() { - this.ownedBuffer = new NativeBufferWriter(); - this.buffer = ownedBuffer; + this.buffer = new NativeBufferWriter(); this.flags = 0; } public QwpWebSocketEncoder(int bufferSize) { - this.ownedBuffer = new NativeBufferWriter(bufferSize); - this.buffer = ownedBuffer; + this.buffer = new NativeBufferWriter(bufferSize); this.flags = 0; } @Override public void close() { - if (ownedBuffer != null) { - ownedBuffer.close(); - ownedBuffer = null; + if (buffer != null) { + buffer.close(); + buffer = null; } } @@ -124,18 +121,8 @@ public boolean isGorillaEnabled() { return (flags & FLAG_GORILLA) != 0; } - public boolean isUsingExternalBuffer() { - return buffer != ownedBuffer; - } - public void reset() { - if (!isUsingExternalBuffer()) { - buffer.reset(); - } - } - - public void setBuffer(QwpBufferWriter externalBuffer) { - this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; + buffer.reset(); } public void setDeltaSymbolDictEnabled(boolean enabled) { From 3f38d628d188c1316a1b5172b4a1b5a8ebc60c16 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 14:03:09 +0100 Subject: [PATCH 126/230] Fix native memory leaks on allocation failure ColumnBuffer constructor leaks allocateStorage() buffers when the subsequent Unsafe.calloc() for the null bitmap throws. Wrap both allocateStorage() and calloc in try/catch so close() frees any partially allocated buffers before rethrowing. Similarly, allocateStorage() leaks stringOffsets when the stringData allocation fails for STRING/VARCHAR columns. Guard the second allocation with try/catch that closes stringOffsets on failure. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 9e929e6..c9f79b0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -437,11 +437,16 @@ public ColumnBuffer(String name, byte type, boolean nullable) { this.valueCount = 0; this.hasNulls = false; - allocateStorage(type); - if (nullable) { - nullBufCapRows = 64; // multiple of 64 - long sizeBytes = (long) nullBufCapRows >>> 3; - nullBufPtr = Unsafe.calloc(sizeBytes, MemoryTag.NATIVE_ILP_RSS); + try { + allocateStorage(type); + if (nullable) { + nullBufCapRows = 64; // multiple of 64 + long sizeBytes = (long) nullBufCapRows >>> 3; + nullBufPtr = Unsafe.calloc(sizeBytes, MemoryTag.NATIVE_ILP_RSS); + } + } catch (Throwable t) { + close(); + throw t; } } @@ -1202,8 +1207,14 @@ private void allocateStorage(byte type) { case TYPE_STRING: case TYPE_VARCHAR: stringOffsets = new OffHeapAppendMemory(64); - stringOffsets.putInt(0); // seed initial 0 offset - stringData = new OffHeapAppendMemory(256); + try { + stringOffsets.putInt(0); // seed initial 0 offset + stringData = new OffHeapAppendMemory(256); + } catch (Throwable t) { + stringOffsets.close(); + stringOffsets = null; + throw t; + } break; case TYPE_SYMBOL: dataBuffer = new OffHeapAppendMemory(64); From 31bc0926c5df00a0650ab37a2f90bc8941e8d9f4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 14:08:18 +0100 Subject: [PATCH 127/230] Fix lone surrogate hash mismatch with wire encoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QwpSchemaHash.computeSchemaHash() and computeSchemaHashDirect() encoded lone surrogates (high surrogate at end of string, or lone low surrogate) as 3-byte UTF-8, while OffHeapAppendMemory.putUtf8() replaced them with '?' (1 byte). This mismatch caused the schema hash to diverge from the actual wire encoding when a column name contained a lone surrogate. Add a Character.isSurrogate(c) guard before the 3-byte else branch in both methods, so lone surrogates hash as '?' — consistent with putUtf8(). Add tests covering lone high surrogate at end of string and lone low surrogate for both computeSchemaHash and computeSchemaHashDirect. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpSchemaHash.java | 4 ++ .../protocol/QwpSchemaHashSurrogateTest.java | 56 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 94b3693..561616c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -122,6 +122,8 @@ public static long computeSchemaHash(String[] columnNames, byte[] columnTypes) { hasher.update((byte) '?'); j--; } + } else if (Character.isSurrogate(c)) { + hasher.update((byte) '?'); } else { // Three bytes hasher.update((byte) (0xE0 | (c >> 12))); @@ -195,6 +197,8 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList> 12))); hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java index 76610f7..ab4d632 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java @@ -51,6 +51,36 @@ public void testComputeSchemaHashInvalidSurrogatePair() { assertEquals(hashExpected, hashInvalid); } + @Test + public void testComputeSchemaHashLoneHighSurrogateAtEnd() { + byte[] types = {TYPE_LONG}; + + // "\uD800" is a lone high surrogate at end of string. + // Must hash as '?' to match OffHeapAppendMemory.putUtf8(). + long hashInvalid = QwpSchemaHash.computeSchemaHash( + new String[]{"col\uD800"}, types + ); + long hashExpected = QwpSchemaHash.computeSchemaHash( + new String[]{"col?"}, types + ); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashLoneLowSurrogate() { + byte[] types = {TYPE_LONG}; + + // "\uDC00" is a lone low surrogate (not preceded by a high surrogate). + // Must hash as '?' to match OffHeapAppendMemory.putUtf8(). + long hashInvalid = QwpSchemaHash.computeSchemaHash( + new String[]{"col\uDC00"}, types + ); + long hashExpected = QwpSchemaHash.computeSchemaHash( + new String[]{"col?"}, types + ); + assertEquals(hashExpected, hashInvalid); + } + @Test public void testComputeSchemaHashDirectInvalidSurrogatePair() { ObjList invalidCols = new ObjList<>(); @@ -63,4 +93,30 @@ public void testComputeSchemaHashDirectInvalidSurrogatePair() { long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); assertEquals(hashExpected, hashInvalid); } + + @Test + public void testComputeSchemaHashDirectLoneHighSurrogateAtEnd() { + ObjList invalidCols = new ObjList<>(); + invalidCols.add(new QwpTableBuffer.ColumnBuffer("col\uD800", TYPE_LONG, false)); + + ObjList expectedCols = new ObjList<>(); + expectedCols.add(new QwpTableBuffer.ColumnBuffer("col?", TYPE_LONG, false)); + + long hashInvalid = QwpSchemaHash.computeSchemaHashDirect(invalidCols); + long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashDirectLoneLowSurrogate() { + ObjList invalidCols = new ObjList<>(); + invalidCols.add(new QwpTableBuffer.ColumnBuffer("col\uDC00", TYPE_LONG, false)); + + ObjList expectedCols = new ObjList<>(); + expectedCols.add(new QwpTableBuffer.ColumnBuffer("col?", TYPE_LONG, false)); + + long hashInvalid = QwpSchemaHash.computeSchemaHashDirect(invalidCols); + long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); + assertEquals(hashExpected, hashInvalid); + } } From dc6833c12299ce4f3ccfcb4f1e2dbe075066e1ea Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 14:34:01 +0100 Subject: [PATCH 128/230] Delete unused code --- .../qwp/websocket/WebSocketCloseCode.java | 19 -- .../qwp/websocket/WebSocketFrameParser.java | 12 +- .../qwp/websocket/WebSocketHandshake.java | 225 ------------------ 3 files changed, 1 insertion(+), 255 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 83253aa..0e70e0c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -144,23 +144,4 @@ public static String describe(int code) { } } - /** - * Checks if a close code is valid for use in a Close frame. - * Codes 1005 and 1006 are reserved and must not be sent. - * - * @param code the close code - * @return true if the code can be sent in a Close frame - */ - public static boolean isValidForSending(int code) { - if (code < 1000) { - return false; - } - if (code == NO_STATUS_RECEIVED || code == ABNORMAL_CLOSURE || code == TLS_HANDSHAKE) { - return false; - } - // 1000-2999 are defined by RFC 6455 - // 3000-3999 are reserved for libraries/frameworks - // 4000-4999 are reserved for applications - return code < 5000; - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index 85603d1..892c7cb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -75,7 +75,6 @@ public class WebSocketFrameParser { private long payloadLength; // Parser state private int state = STATE_HEADER; - private boolean strictMode = false; // If true, reject non-minimal length encodings public int getErrorCode() { return errorCode; @@ -164,6 +163,7 @@ public int parse(long buf, long limit) { // Calculate header size and payload length int offset = 2; + // If true, reject non-minimal length encodings if (lengthField <= 125) { payloadLength = lengthField; } else if (lengthField == 126) { @@ -177,11 +177,6 @@ public int parse(long buf, long limit) { payloadLength = (high << 8) | low; // Strict mode: reject non-minimal encodings - if (strictMode && payloadLength < 126) { - state = STATE_ERROR; - errorCode = WebSocketCloseCode.PROTOCOL_ERROR; - return 0; - } offset = 4; } else { @@ -193,11 +188,6 @@ public int parse(long buf, long limit) { payloadLength = Long.reverseBytes(Unsafe.getUnsafe().getLong(buf + 2)); // Strict mode: reject non-minimal encodings - if (strictMode && payloadLength <= 65535) { - state = STATE_ERROR; - errorCode = WebSocketCloseCode.PROTOCOL_ERROR; - return 0; - } // MSB must be 0 (no negative lengths) if (payloadLength < 0) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java index e87cb1d..650fc9f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java @@ -24,7 +24,6 @@ package io.questdb.client.cutlass.qwp.websocket; -import io.questdb.client.std.Unsafe; import io.questdb.client.std.str.Utf8Sequence; import io.questdb.client.std.str.Utf8String; import io.questdb.client.std.str.Utf8s; @@ -40,15 +39,6 @@ * generating proper handshake responses. */ public final class WebSocketHandshake { - public static final Utf8String HEADER_CONNECTION = new Utf8String("Connection"); - public static final Utf8String HEADER_SEC_WEBSOCKET_ACCEPT = new Utf8String("Sec-WebSocket-Accept"); - public static final Utf8String HEADER_SEC_WEBSOCKET_KEY = new Utf8String("Sec-WebSocket-Key"); - public static final Utf8String HEADER_SEC_WEBSOCKET_PROTOCOL = new Utf8String("Sec-WebSocket-Protocol"); - public static final Utf8String HEADER_SEC_WEBSOCKET_VERSION = new Utf8String("Sec-WebSocket-Version"); - // Header names (case-insensitive) - public static final Utf8String HEADER_UPGRADE = new Utf8String("Upgrade"); - public static final Utf8String VALUE_UPGRADE = new Utf8String("upgrade"); - // Header values public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); /** * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. @@ -58,12 +48,6 @@ public final class WebSocketHandshake { * The required WebSocket version (RFC 6455). */ public static final int WEBSOCKET_VERSION = 13; - // Response template - private static final byte[] RESPONSE_PREFIX = - "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ".getBytes(StandardCharsets.US_ASCII); - private static final byte[] RESPONSE_SUFFIX = "\r\n\r\n".getBytes(StandardCharsets.US_ASCII); - - // Thread-local SHA-1 digest for computing Sec-WebSocket-Accept private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { try { return MessageDigest.getInstance("SHA-1"); @@ -76,29 +60,6 @@ private WebSocketHandshake() { // Static utility class } - /** - * Computes the Sec-WebSocket-Accept value for the given key. - * - * @param key the Sec-WebSocket-Key from the client - * @return the base64-encoded SHA-1 hash to send in the response - */ - public static String computeAcceptKey(Utf8Sequence key) { - MessageDigest sha1 = SHA1_DIGEST.get(); - sha1.reset(); - - // Concatenate key + GUID - byte[] keyBytes = new byte[key.size()]; - for (int i = 0; i < key.size(); i++) { - keyBytes[i] = key.byteAt(i); - } - sha1.update(keyBytes); - sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); - - // Compute SHA-1 hash and base64 encode - byte[] hash = sha1.digest(); - return Base64.getEncoder().encodeToString(hash); - } - /** * Computes the Sec-WebSocket-Accept value for the given key string. * @@ -198,31 +159,6 @@ public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); } - /** - * Returns the size of the handshake response for the given accept key. - * - * @param acceptKey the computed accept key - * @return the total response size in bytes - */ - public static int responseSize(String acceptKey) { - return RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; - } - - /** - * Returns the size of the handshake response with an optional subprotocol. - * - * @param acceptKey the computed accept key - * @param protocol the negotiated subprotocol (may be null or empty) - * @return the total response size in bytes - */ - public static int responseSizeWithProtocol(String acceptKey, String protocol) { - int size = RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; - if (protocol != null && !protocol.isEmpty()) { - size += "\r\nSec-WebSocket-Protocol: ".length() + protocol.length(); - } - return size; - } - /** * Validates all required headers for a WebSocket upgrade request. * @@ -253,165 +189,4 @@ public static String validate( return null; } - /** - * Writes a 400 Bad Request response. - * - * @param buf the buffer to write to - * @param reason the reason for the bad request - * @return the number of bytes written - */ - public static int writeBadRequestResponse(long buf, String reason) { - int offset = 0; - - byte[] statusLine = "HTTP/1.1 400 Bad Request\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : statusLine) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] contentType = "Content-Type: text/plain\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : contentType) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] reasonBytes = reason != null ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; - byte[] contentLength = ("Content-Length: " + reasonBytes.length + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII); - for (byte b : contentLength) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - for (byte b : reasonBytes) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - return offset; - } - - /** - * Writes the WebSocket handshake response to the given buffer. - * - * @param buf the buffer to write to - * @param acceptKey the computed Sec-WebSocket-Accept value - * @return the number of bytes written - */ - public static int writeResponse(long buf, String acceptKey) { - int offset = 0; - - // Write prefix - for (byte b : RESPONSE_PREFIX) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - // Write accept key - byte[] acceptBytes = acceptKey.getBytes(StandardCharsets.US_ASCII); - for (byte b : acceptBytes) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - // Write suffix - for (byte b : RESPONSE_SUFFIX) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - return offset; - } - - /** - * Writes the WebSocket handshake response with an optional subprotocol. - * - * @param buf the buffer to write to - * @param acceptKey the computed Sec-WebSocket-Accept value - * @param protocol the negotiated subprotocol (may be null or empty) - * @return the number of bytes written - */ - public static int writeResponseWithProtocol(long buf, String acceptKey, String protocol) { - int offset = 0; - - // Write prefix - for (byte b : RESPONSE_PREFIX) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - // Write accept key - byte[] acceptBytes = acceptKey.getBytes(StandardCharsets.US_ASCII); - for (byte b : acceptBytes) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - // Write protocol header if present - if (protocol != null && !protocol.isEmpty()) { - byte[] protocolHeader = ("\r\nSec-WebSocket-Protocol: " + protocol).getBytes(StandardCharsets.US_ASCII); - for (byte b : protocolHeader) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - } - - // Write suffix - for (byte b : RESPONSE_SUFFIX) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - return offset; - } - - /** - * Writes a 426 Upgrade Required response indicating unsupported WebSocket version. - * - * @param buf the buffer to write to - * @return the number of bytes written - */ - public static int writeVersionNotSupportedResponse(long buf) { - int offset = 0; - - byte[] statusLine = "HTTP/1.1 426 Upgrade Required\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : statusLine) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] versionHeader = "Sec-WebSocket-Version: 13\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : versionHeader) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] contentLength = "Content-Length: 0\r\n\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : contentLength) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - return offset; - } - - /** - * Checks if the sequence contains the given substring (case-insensitive). - */ - private static boolean containsIgnoreCaseAscii(Utf8Sequence seq, Utf8Sequence substring) { - int seqLen = seq.size(); - int subLen = substring.size(); - - if (subLen > seqLen) { - return false; - } - if (subLen == 0) { - return true; - } - - outer: - for (int i = 0; i <= seqLen - subLen; i++) { - for (int j = 0; j < subLen; j++) { - byte a = seq.byteAt(i + j); - byte b = substring.byteAt(j); - // Convert to lowercase for comparison - if (a >= 'A' && a <= 'Z') { - a = (byte) (a + 32); - } - if (b >= 'A' && b <= 'Z') { - b = (byte) (b + 32); - } - if (a != b) { - continue outer; - } - } - return true; - } - return false; - } } From 53e448185f29ba81909ac8031ea18cc60bc5fedb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 14:34:23 +0100 Subject: [PATCH 129/230] Use enhanced switch --- .../qwp/websocket/WebSocketCloseCode.java | 52 +++++++------------ .../qwp/websocket/WebSocketOpcode.java | 25 ++++----- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 0e70e0c..6ee2071 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -107,41 +107,29 @@ private WebSocketCloseCode() { * @return the description */ public static String describe(int code) { - switch (code) { - case NORMAL_CLOSURE: - return "Normal Closure"; - case GOING_AWAY: - return "Going Away"; - case PROTOCOL_ERROR: - return "Protocol Error"; - case UNSUPPORTED_DATA: - return "Unsupported Data"; - case RESERVED: - return "Reserved"; - case NO_STATUS_RECEIVED: - return "No Status Received"; - case ABNORMAL_CLOSURE: - return "Abnormal Closure"; - case INVALID_PAYLOAD_DATA: - return "Invalid Payload Data"; - case POLICY_VIOLATION: - return "Policy Violation"; - case MESSAGE_TOO_BIG: - return "Message Too Big"; - case MANDATORY_EXTENSION: - return "Mandatory Extension"; - case INTERNAL_ERROR: - return "Internal Error"; - case TLS_HANDSHAKE: - return "TLS Handshake"; - default: + return switch (code) { + case NORMAL_CLOSURE -> "Normal Closure"; + case GOING_AWAY -> "Going Away"; + case PROTOCOL_ERROR -> "Protocol Error"; + case UNSUPPORTED_DATA -> "Unsupported Data"; + case RESERVED -> "Reserved"; + case NO_STATUS_RECEIVED -> "No Status Received"; + case ABNORMAL_CLOSURE -> "Abnormal Closure"; + case INVALID_PAYLOAD_DATA -> "Invalid Payload Data"; + case POLICY_VIOLATION -> "Policy Violation"; + case MESSAGE_TOO_BIG -> "Message Too Big"; + case MANDATORY_EXTENSION -> "Mandatory Extension"; + case INTERNAL_ERROR -> "Internal Error"; + case TLS_HANDSHAKE -> "TLS Handshake"; + default -> { if (code >= 3000 && code < 4000) { - return "Library/Framework Code (" + code + ")"; + yield "Library/Framework Code (" + code + ")"; } else if (code >= 4000 && code < 5000) { - return "Application Code (" + code + ")"; + yield "Application Code (" + code + ")"; } - return "Unknown (" + code + ")"; - } + yield "Unknown (" + code + ")"; + } + }; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java index f2fead7..31f1cc8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -111,21 +111,14 @@ public static boolean isValid(int opcode) { * @return the opcode name */ public static String name(int opcode) { - switch (opcode) { - case CONTINUATION: - return "CONTINUATION"; - case TEXT: - return "TEXT"; - case BINARY: - return "BINARY"; - case CLOSE: - return "CLOSE"; - case PING: - return "PING"; - case PONG: - return "PONG"; - default: - return "UNKNOWN(" + opcode + ")"; - } + return switch (opcode) { + case CONTINUATION -> "CONTINUATION"; + case TEXT -> "TEXT"; + case BINARY -> "BINARY"; + case CLOSE -> "CLOSE"; + case PING -> "PING"; + case PONG -> "PONG"; + default -> "UNKNOWN(" + opcode + ")"; + }; } } From b2cd9a81710cd46bd91d6755de5bcec0b9f9a53e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 14:36:11 +0100 Subject: [PATCH 130/230] Simplify websocket handshake code --- .../qwp/websocket/WebSocketHandshake.java | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java index 650fc9f..b79e7b4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java @@ -40,10 +40,6 @@ */ public final class WebSocketHandshake { public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); - /** - * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. - */ - public static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; /** * The required WebSocket version (RFC 6455). */ @@ -55,6 +51,10 @@ public final class WebSocketHandshake { throw new RuntimeException("SHA-1 not available", e); } }); + /** + * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. + */ + private static final byte[] WEBSOCKET_GUID_BYTES = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11".getBytes(StandardCharsets.US_ASCII); private WebSocketHandshake() { // Static utility class @@ -70,9 +70,10 @@ public static String computeAcceptKey(String key) { MessageDigest sha1 = SHA1_DIGEST.get(); sha1.reset(); - // Concatenate key + GUID - sha1.update(key.getBytes(StandardCharsets.US_ASCII)); - sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + for (int i = 0, n = key.length(); i < n; i++) { + sha1.update((byte) key.charAt(i)); + } + sha1.update(WEBSOCKET_GUID_BYTES); // Compute SHA-1 hash and base64 encode byte[] hash = sha1.digest(); @@ -91,7 +92,23 @@ public static boolean isConnectionUpgrade(Utf8Sequence connectionHeader) { } // Connection header may contain multiple values, e.g., "keep-alive, Upgrade" // Perform case-insensitive substring search - return containsIgnoreCaseAscii(connectionHeader, VALUE_UPGRADE); + int seqLen = connectionHeader.size(); + if (seqLen < 7) { + return false; + } + for (int i = 0; i <= seqLen - 7; i++) { + if ((connectionHeader.byteAt(i) | 32) == 'u' + && (connectionHeader.byteAt(i + 1) | 32) == 'p' + && (connectionHeader.byteAt(i + 2) | 32) == 'g' + && (connectionHeader.byteAt(i + 3) | 32) == 'r' + && (connectionHeader.byteAt(i + 4) | 32) == 'a' + && (connectionHeader.byteAt(i + 5) | 32) == 'd' + && (connectionHeader.byteAt(i + 6) | 32) == 'e' + ) { + return true; + } + } + return false; } /** From 99b7552d37fa49b6e1c13c11654dddd9401469ef Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 15:03:22 +0100 Subject: [PATCH 131/230] Code style, remove dead code --- .../qwp/client/QwpWebSocketSender.java | 57 ++++++++----------- .../cutlass/qwp/protocol/QwpBitWriter.java | 21 +------ .../qwp/protocol/QwpGorillaEncoder.java | 39 ++++++------- .../cutlass/qwp/protocol/QwpTableBuffer.java | 52 ++++++----------- .../qwp/websocket/WebSocketFrameParser.java | 17 ++---- 5 files changed, 61 insertions(+), 125 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 9981c36..e865167 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -113,10 +113,13 @@ public class QwpWebSocketSender implements Sender { private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final String WRITE_PATH = "/write/v4"; + private final AckFrameHandler ackHandler = new AckFrameHandler(this); + private final WebSocketResponse ackResponse = new WebSocketResponse(); private final int autoFlushBytes; private final long autoFlushIntervalNanos; // Auto-flush configuration private final int autoFlushRows; + private final Decimal256 currentDecimal256 = new Decimal256(); // Encoder for ILP v4 messages private final QwpWebSocketEncoder encoder; // Global symbol dictionary for delta encoding @@ -131,8 +134,6 @@ public class QwpWebSocketSender implements Sender { private final LongHashSet sentSchemaHashes = new LongHashSet(); private final CharSequenceObjHashMap tableBuffers; private final boolean tlsEnabled; - private final AckFrameHandler ackHandler = new AckFrameHandler(this); - private final WebSocketResponse ackResponse = new WebSocketResponse(); private MicrobatchBuffer activeBuffer; // Double-buffering for async I/O private MicrobatchBuffer buffer0; @@ -146,7 +147,6 @@ public class QwpWebSocketSender implements Sender { private boolean connected; // Track max global symbol ID used in current batch (for delta calculation) private int currentBatchMaxSymbolId = -1; - private final Decimal256 currentDecimal256 = new Decimal256(); private QwpTableBuffer currentTableBuffer; private String currentTableName; private long firstPendingRowTimeNanos; @@ -710,14 +710,6 @@ public int getMaxSentSymbolId() { return maxSentSymbolId; } - /** - * Returns the number of pending rows not yet flushed. - * For testing. - */ - public int getPendingRowCount() { - return pendingRowCount; - } - /** * Registers a symbol in the global dictionary and returns its ID. * For use with fast-path column buffer access. @@ -730,6 +722,14 @@ public int getOrAddGlobalSymbol(String value) { return globalId; } + /** + * Returns the number of pending rows not yet flushed. + * For testing. + */ + public int getPendingRowCount() { + return pendingRowCount; + } + /** * Gets or creates a table buffer for direct access. * For high-throughput generators that want to bypass fluent API overhead. @@ -886,10 +886,9 @@ public void reset() { /** * Sets whether to use Gorilla timestamp encoding. */ - public QwpWebSocketSender setGorillaEnabled(boolean enabled) { + public void setGorillaEnabled(boolean enabled) { this.gorillaEnabled = enabled; this.encoder.setGorillaEnabled(enabled); - return this; } /** @@ -1382,33 +1381,23 @@ private boolean shouldAutoFlush() { // Time limit if (autoFlushIntervalNanos > 0) { long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; - if (ageNanos >= autoFlushIntervalNanos) { - return true; - } + return ageNanos >= autoFlushIntervalNanos; } // Byte limit is harder to estimate without encoding, skip for now return false; } private long toMicros(long value, ChronoUnit unit) { - switch (unit) { - case NANOS: - return value / 1000L; - case MICROS: - return value; - case MILLIS: - return value * 1000L; - case SECONDS: - return value * 1_000_000L; - case MINUTES: - return value * 60_000_000L; - case HOURS: - return value * 3_600_000_000L; - case DAYS: - return value * 86_400_000_000L; - default: - throw new LineSenderException("Unsupported time unit: " + unit); - } + return switch (unit) { + case NANOS -> value / 1000L; + case MICROS -> value; + case MILLIS -> value * 1000L; + case SECONDS -> value * 1_000_000L; + case MINUTES -> value * 60_000_000L; + case HOURS -> value * 3_600_000_000L; + case DAYS -> value * 86_400_000_000L; + default -> throw new LineSenderException("Unsupported time unit: " + unit); + }; } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index 30173cb..5e217f1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -103,16 +103,6 @@ public void flush() { } } - /** - * Returns the number of bits remaining in the partial byte buffer. - * This is 0 after a flush or when aligned on a byte boundary. - * - * @return bits in buffer (0-7) - */ - public int getBitsInBuffer() { - return bitsInBuffer; - } - /** * Returns the current write position (address). * Note: Call {@link #flush()} first to ensure all buffered bits are written. @@ -123,15 +113,6 @@ public long getPosition() { return currentAddress; } - /** - * Returns the number of bits that have been written (including buffered bits). - * - * @return total bits written since reset - */ - public long getTotalBitsWritten() { - return (currentAddress - startAddress) * 8L + bitsInBuffer; - } - /** * Resets the writer to write to the specified memory region. * @@ -166,7 +147,7 @@ public void writeBit(int bit) { */ public void writeBits(long value, int numBits) { if (numBits <= 0 || numBits > 64) { - return; + throw new AssertionError("Asked to write more than 64 bits of a long"); } // Mask the value to only include the requested bits diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 0f57342..302299c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -148,18 +148,13 @@ public static boolean canUseGorilla(long srcAddress, int count) { */ public static int getBitsRequired(long deltaOfDelta) { int bucket = getBucket(deltaOfDelta); - switch (bucket) { - case 0: - return 1; - case 1: - return 9; - case 2: - return 12; - case 3: - return 16; - default: - return 36; - } + return switch (bucket) { + case 0 -> 1; + case 1 -> 9; + case 2 -> 12; + case 3 -> 16; + default -> 36; + }; } /** @@ -199,25 +194,23 @@ public static int getBucket(long deltaOfDelta) { public void encodeDoD(long deltaOfDelta) { int bucket = getBucket(deltaOfDelta); switch (bucket) { - case 0: // DoD == 0 - bitWriter.writeBit(0); - break; - case 1: // [-64, 63] -> '10' + 7-bit + case 0 -> bitWriter.writeBit(0); + case 1 -> { bitWriter.writeBits(0b01, 2); bitWriter.writeSigned(deltaOfDelta, 7); - break; - case 2: // [-256, 255] -> '110' + 9-bit + } + case 2 -> { bitWriter.writeBits(0b011, 3); bitWriter.writeSigned(deltaOfDelta, 9); - break; - case 3: // [-2048, 2047] -> '1110' + 12-bit + } + case 3 -> { bitWriter.writeBits(0b0111, 4); bitWriter.writeSigned(deltaOfDelta, 12); - break; - default: // '1111' + 32-bit + } + default -> { bitWriter.writeBits(0b1111, 4); bitWriter.writeSigned(deltaOfDelta, 32); - break; + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index c9f79b0..6d83c89 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -271,34 +271,16 @@ public void reset() { * @see QwpConstants#getFixedTypeSize(byte) for wire-format sizes */ static int elementSizeInBuffer(byte type) { - switch (type) { - case TYPE_BOOLEAN: - case TYPE_BYTE: - return 1; - case TYPE_SHORT: - case TYPE_CHAR: - return 2; - case TYPE_INT: - case TYPE_SYMBOL: - case TYPE_FLOAT: - return 4; - case TYPE_GEOHASH: - case TYPE_LONG: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: - case TYPE_DECIMAL64: - case TYPE_DOUBLE: - return 8; - case TYPE_UUID: - case TYPE_DECIMAL128: - return 16; - case TYPE_LONG256: - case TYPE_DECIMAL256: - return 32; - default: - return 0; - } + return switch (type) { + case TYPE_BOOLEAN, TYPE_BYTE -> 1; + case TYPE_SHORT, TYPE_CHAR -> 2; + case TYPE_INT, TYPE_SYMBOL, TYPE_FLOAT -> 4; + case TYPE_GEOHASH, TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, + TYPE_DATE, TYPE_DECIMAL64, TYPE_DOUBLE -> 8; + case TYPE_UUID, TYPE_DECIMAL128 -> 16; + case TYPE_LONG256, TYPE_DECIMAL256 -> 32; + default -> 0; + }; } /** @@ -1173,6 +1155,13 @@ public void truncateTo(int newSize) { } } + private static int checkedElementCount(long product) { + if (product > Integer.MAX_VALUE) { + throw new LineSenderException("array too large: total element count exceeds int range"); + } + return (int) product; + } + private void allocateStorage(byte type) { switch (type) { case TYPE_BOOLEAN: @@ -1229,13 +1218,6 @@ private void allocateStorage(byte type) { } } - private static int checkedElementCount(long product) { - if (product > Integer.MAX_VALUE) { - throw new LineSenderException("array too large: total element count exceeds int range"); - } - return (int) product; - } - private void ensureArrayCapacity(int nDims, int dataElements) { // Ensure per-row array dims capacity if (valueCount >= arrayDims.length) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index 892c7cb..ffd8a37 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -147,7 +147,8 @@ public int parse(long buf, long limit) { return 0; } - masked = (byte1 & MASK_BIT) != 0; + final boolean masked = (byte1 & MASK_BIT) != 0; + this.masked = masked; int lengthField = byte1 & LENGTH_MASK; // Validate masking based on mode @@ -213,18 +214,7 @@ public int parse(long buf, long limit) { return 0; } - // Parse mask key if present - if (masked) { - if (available < offset + 4) { - state = STATE_NEED_MORE; - return 0; - } - maskKey = Unsafe.getUnsafe().getInt(buf + offset); - offset += 4; - } else { - maskKey = 0; - } - + maskKey = 0; headerSize = offset; // Check if we have the complete payload @@ -260,6 +250,7 @@ public void reset() { */ public void unmaskPayload(long buf, long len) { if (!masked || maskKey == 0) { + // a zero maskKey is a no-op (makes no change to the data) return; } From 5df3ca5e8e452bdba5c7b2bd16629ad756d715fa Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 15:14:41 +0100 Subject: [PATCH 132/230] Delete dead code, enhanced switch, auto-reorder --- .../cutlass/http/client/WebSocketClient.java | 47 ++-- .../cutlass/qwp/client/MicrobatchBuffer.java | 19 +- .../qwp/client/QwpWebSocketEncoder.java | 61 +++-- .../qwp/client/QwpWebSocketSender.java | 9 +- .../cutlass/qwp/client/WebSocketResponse.java | 33 +-- .../qwp/client/WebSocketSendQueue.java | 35 --- .../cutlass/qwp/protocol/QwpSchemaHash.java | 26 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 19 +- .../qwp/websocket/WebSocketHandshake.java | 209 ------------------ .../qwp/websocket/WebSocketOpcode.java | 17 -- 10 files changed, 83 insertions(+), 392 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 375829c..84c4329 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -27,7 +27,6 @@ import io.questdb.client.HttpClientConfiguration; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; -import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.network.IOOperation; import io.questdb.client.network.NetworkFacade; @@ -44,6 +43,8 @@ import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Base64; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -70,6 +71,14 @@ public abstract class WebSocketClient implements QuietCloseable { private static final int DEFAULT_RECV_BUFFER_SIZE = 65536; private static final int DEFAULT_SEND_BUFFER_SIZE = 65536; private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class); + private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-1 not available", e); + } + }); + private static final byte[] WEBSOCKET_GUID_BYTES = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11".getBytes(StandardCharsets.US_ASCII); protected final NetworkFacade nf; protected final Socket socket; private final WebSocketSendBuffer controlFrameBuffer; @@ -435,6 +444,16 @@ public void upgrade(CharSequence path) { upgrade(path, defaultTimeout); } + private static String computeAcceptKey(String key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + for (int i = 0, n = key.length(); i < n; i++) { + sha1.update((byte) key.charAt(i)); + } + sha1.update(WEBSOCKET_GUID_BYTES); + return Base64.getEncoder().encodeToString(sha1.digest()); + } + private static boolean containsHeaderValue(String response, String headerName, String expectedValue, boolean ignoreValueCase) { int headerLen = headerName.length(); int responseLen = response.length(); @@ -454,6 +473,10 @@ private static boolean containsHeaderValue(String response, String headerName, S return false; } + private static int remainingTime(int timeoutMillis, long startTimeNanos) { + return timeoutMillis - (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); + } + private void appendToFragmentBuffer(long payloadPtr, int payloadLen) { if (payloadLen == 0) { return; @@ -588,6 +611,14 @@ private int findHeaderEnd() { return -1; } + private int getRemainingTimeOrThrow(int timeoutMillis, long startTimeNanos) { + int remaining = remainingTime(timeoutMillis, startTimeNanos); + if (remaining <= 0) { + throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']').flagAsTimeout(); + } + return remaining; + } + private void growRecvBuffer() { int newSize = (int) Math.min((long) recvBufSize * 2, maxRecvBufSize); if (newSize >= maxRecvBufSize) { @@ -668,18 +699,6 @@ private int recvOrTimeout(long ptr, int len, int timeout) { return n; } - private int getRemainingTimeOrThrow(int timeoutMillis, long startTimeNanos) { - int remaining = remainingTime(timeoutMillis, startTimeNanos); - if (remaining <= 0) { - throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']').flagAsTimeout(); - } - return remaining; - } - - private static int remainingTime(int timeoutMillis, long startTimeNanos) { - return timeoutMillis - (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); - } - private void resetFragmentState() { fragmentOpcode = -1; fragmentBufPos = 0; @@ -852,7 +871,7 @@ private void validateUpgradeResponse(int headerEnd) { } // Verify Sec-WebSocket-Accept (exact value match per RFC 6455 Section 4.1) - String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey); + String expectedAccept = computeAcceptKey(handshakeKey); if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept, false)) { throw new HttpClientException("Invalid Sec-WebSocket-Accept header"); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 41a3310..94cb89e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -118,18 +118,13 @@ public MicrobatchBuffer(int initialCapacity) { * Returns a human-readable name for the given state. */ public static String stateName(int state) { - switch (state) { - case STATE_FILLING: - return "FILLING"; - case STATE_SEALED: - return "SEALED"; - case STATE_SENDING: - return "SENDING"; - case STATE_RECYCLED: - return "RECYCLED"; - default: - return "UNKNOWN(" + state + ")"; - } + return switch (state) { + case STATE_FILLING -> "FILLING"; + case STATE_SEALED -> "SEALED"; + case STATE_SENDING -> "SENDING"; + case STATE_RECYCLED -> "RECYCLED"; + default -> "UNKNOWN(" + state + ")"; + }; } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 9a5f7ea..34f3659 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -171,25 +171,16 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, case TYPE_CHAR: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); break; - case TYPE_INT: + case TYPE_INT, TYPE_FLOAT: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; - case TYPE_LONG: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_FLOAT: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); - break; - case TYPE_DOUBLE: + case TYPE_LONG, TYPE_DATE, TYPE_DOUBLE: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: writeTimestampColumn(dataAddr, valueCount, useGorilla); break; - case TYPE_DATE: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; case TYPE_GEOHASH: writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); break; @@ -314,30 +305,6 @@ private void writeDecimal64Column(byte scale, long addr, int count) { } } - /** - * Writes a GeoHash column in variable-width wire format. - *

    - * Wire format: [precision varint] [packed values: ceil(precision/8) bytes each] - * Values are stored as 8-byte longs in the off-heap buffer but only the - * lower ceil(precision/8) bytes are written to the wire. - */ - private void writeGeoHashColumn(long addr, int count, int precision) { - if (precision < 1) { - // All values are null: use minimum valid precision. - // The decoder will skip all values via the null bitmap, - // so the precision only needs to be structurally valid. - precision = 1; - } - buffer.putVarint(precision); - int valueSize = (precision + 7) / 8; - for (int i = 0; i < count; i++) { - long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8); - for (int b = 0; b < valueSize; b++) { - buffer.putByte((byte) (value >>> (b * 8))); - } - } - } - private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); @@ -362,6 +329,30 @@ private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) } } + /** + * Writes a GeoHash column in variable-width wire format. + *

    + * Wire format: [precision varint] [packed values: ceil(precision/8) bytes each] + * Values are stored as 8-byte longs in the off-heap buffer but only the + * lower ceil(precision/8) bytes are written to the wire. + */ + private void writeGeoHashColumn(long addr, int count, int precision) { + if (precision < 1) { + // All values are null: use minimum valid precision. + // The decoder will skip all values via the null bitmap, + // so the precision only needs to be structurally valid. + precision = 1; + } + buffer.putVarint(precision); + int valueSize = (precision + 7) / 8; + for (int i = 0; i < count; i++) { + long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8); + for (int b = 0; b < valueSize; b++) { + buffer.putByte((byte) (value >>> (b * 8))); + } + } + } + private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index e865167..723c422 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1450,12 +1450,9 @@ private void waitForAck(long expectedSequence) { throw timeout; } - private static class AckFrameHandler implements WebSocketFrameHandler { - private final QwpWebSocketSender sender; - - AckFrameHandler(QwpWebSocketSender sender) { - this.sender = sender; - } + private record AckFrameHandler( + QwpWebSocketSender sender + ) implements WebSocketFrameHandler { @Override public void onBinaryMessage(long payloadPtr, int payloadLen) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 169f419..0070a5e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -141,33 +141,19 @@ public long getSequence() { return sequence; } - /** - * Returns the status code. - */ - public byte getStatus() { - return status; - } - /** * Returns a human-readable status name. */ public String getStatusName() { - switch (status) { - case STATUS_OK: - return "OK"; - case STATUS_PARSE_ERROR: - return "PARSE_ERROR"; - case STATUS_SCHEMA_ERROR: - return "SCHEMA_ERROR"; - case STATUS_WRITE_ERROR: - return "WRITE_ERROR"; - case STATUS_SECURITY_ERROR: - return "SECURITY_ERROR"; - case STATUS_INTERNAL_ERROR: - return "INTERNAL_ERROR"; - default: - return "UNKNOWN(" + (status & 0xFF) + ")"; - } + return switch (status) { + case STATUS_OK -> "OK"; + case STATUS_PARSE_ERROR -> "PARSE_ERROR"; + case STATUS_SCHEMA_ERROR -> "SCHEMA_ERROR"; + case STATUS_WRITE_ERROR -> "WRITE_ERROR"; + case STATUS_SECURITY_ERROR -> "SECURITY_ERROR"; + case STATUS_INTERNAL_ERROR -> "INTERNAL_ERROR"; + default -> "UNKNOWN(" + (status & 0xFF) + ")"; + }; } /** @@ -210,7 +196,6 @@ public boolean readFrom(long ptr, int length) { msgBytes[i] = Unsafe.getUnsafe().getByte(ptr + offset + i); } errorMessage = new String(msgBytes, StandardCharsets.UTF_8); - offset += msgLen; } else { errorMessage = null; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index 7b43fd9..b155bab 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -321,20 +321,6 @@ public Throwable getLastError() { return lastError; } - /** - * Returns the number of batches waiting to be sent. - */ - public int getPendingCount() { - return getPendingSize(); - } - - /** - * Returns total successful acknowledgments received. - */ - public long getTotalAcks() { - return totalAcks.get(); - } - /** * Returns the total number of batches sent. */ @@ -349,27 +335,6 @@ public long getTotalBytesSent() { return totalBytesSent.get(); } - /** - * Returns total error responses received. - */ - public long getTotalErrors() { - return totalErrors.get(); - } - - /** - * Returns true if the queue is empty. - */ - public boolean isEmpty() { - return isPendingEmpty(); - } - - /** - * Returns true if the queue is still running. - */ - public boolean isRunning() { - return running && !shuttingDown; - } - /** * Checks if an error occurred in the I/O thread and throws if so. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 561616c..07f5c38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -26,7 +26,6 @@ import io.questdb.client.std.Unsafe; -import io.questdb.client.std.str.DirectUtf8Sequence; import io.questdb.client.std.str.Utf8Sequence; /** @@ -137,31 +136,6 @@ public static long computeSchemaHash(String[] columnNames, byte[] columnTypes) { return hasher.getValue(); } - /** - * Computes the schema hash for ILP v4 using DirectUtf8Sequence column names. - * - * @param columnNames array of column names - * @param columnTypes array of type codes - * @return the schema hash - */ - public static long computeSchemaHash(DirectUtf8Sequence[] columnNames, byte[] columnTypes) { - // Use pooled hasher to avoid allocation - Hasher hasher = HASHER_POOL.get(); - hasher.reset(DEFAULT_SEED); - - for (int i = 0; i < columnNames.length; i++) { - DirectUtf8Sequence name = columnNames[i]; - long addr = name.ptr(); - int len = name.size(); - for (int j = 0; j < len; j++) { - hasher.update(Unsafe.getUnsafe().getByte(addr + j)); - } - hasher.update(columnTypes[i]); - } - - return hasher.getValue(); - } - /** * Computes the schema hash directly from column buffers without intermediate arrays. * This is the most efficient method when column data is already available. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 6d83c89..783b7ea 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -782,14 +782,10 @@ public void addNull() { } else { // For non-nullable columns, store a sentinel/default value switch (type) { - case TYPE_BOOLEAN: + case TYPE_BOOLEAN, TYPE_BYTE: dataBuffer.putByte((byte) 0); break; - case TYPE_BYTE: - dataBuffer.putByte((byte) 0); - break; - case TYPE_SHORT: - case TYPE_CHAR: + case TYPE_SHORT, TYPE_CHAR: dataBuffer.putShort((short) 0); break; case TYPE_INT: @@ -798,10 +794,7 @@ public void addNull() { case TYPE_GEOHASH: dataBuffer.putLong(-1L); break; - case TYPE_LONG: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: + case TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE: dataBuffer.putLong(Long.MIN_VALUE); break; case TYPE_FLOAT: @@ -810,8 +803,7 @@ public void addNull() { case TYPE_DOUBLE: dataBuffer.putDouble(Double.NaN); break; - case TYPE_STRING: - case TYPE_VARCHAR: + case TYPE_STRING, TYPE_VARCHAR: stringOffsets.putInt((int) stringData.getAppendOffset()); break; case TYPE_SYMBOL: @@ -840,8 +832,7 @@ public void addNull() { dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); break; - case TYPE_DOUBLE_ARRAY: - case TYPE_LONG_ARRAY: + case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY: ensureArrayCapacity(1, 0); arrayDims[valueCount] = 1; arrayShapes[arrayShapeOffset++] = 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java deleted file mode 100644 index b79e7b4..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java +++ /dev/null @@ -1,209 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.websocket; - -import io.questdb.client.std.str.Utf8Sequence; -import io.questdb.client.std.str.Utf8String; -import io.questdb.client.std.str.Utf8s; - -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Base64; - -/** - * WebSocket handshake processing as defined in RFC 6455. - * Provides utilities for validating WebSocket upgrade requests and - * generating proper handshake responses. - */ -public final class WebSocketHandshake { - public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); - /** - * The required WebSocket version (RFC 6455). - */ - public static final int WEBSOCKET_VERSION = 13; - private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { - try { - return MessageDigest.getInstance("SHA-1"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("SHA-1 not available", e); - } - }); - /** - * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. - */ - private static final byte[] WEBSOCKET_GUID_BYTES = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11".getBytes(StandardCharsets.US_ASCII); - - private WebSocketHandshake() { - // Static utility class - } - - /** - * Computes the Sec-WebSocket-Accept value for the given key string. - * - * @param key the Sec-WebSocket-Key from the client - * @return the base64-encoded SHA-1 hash to send in the response - */ - public static String computeAcceptKey(String key) { - MessageDigest sha1 = SHA1_DIGEST.get(); - sha1.reset(); - - for (int i = 0, n = key.length(); i < n; i++) { - sha1.update((byte) key.charAt(i)); - } - sha1.update(WEBSOCKET_GUID_BYTES); - - // Compute SHA-1 hash and base64 encode - byte[] hash = sha1.digest(); - return Base64.getEncoder().encodeToString(hash); - } - - /** - * Checks if the Connection header contains "upgrade". - * - * @param connectionHeader the value of the Connection header - * @return true if the connection should be upgraded - */ - public static boolean isConnectionUpgrade(Utf8Sequence connectionHeader) { - if (connectionHeader == null) { - return false; - } - // Connection header may contain multiple values, e.g., "keep-alive, Upgrade" - // Perform case-insensitive substring search - int seqLen = connectionHeader.size(); - if (seqLen < 7) { - return false; - } - for (int i = 0; i <= seqLen - 7; i++) { - if ((connectionHeader.byteAt(i) | 32) == 'u' - && (connectionHeader.byteAt(i + 1) | 32) == 'p' - && (connectionHeader.byteAt(i + 2) | 32) == 'g' - && (connectionHeader.byteAt(i + 3) | 32) == 'r' - && (connectionHeader.byteAt(i + 4) | 32) == 'a' - && (connectionHeader.byteAt(i + 5) | 32) == 'd' - && (connectionHeader.byteAt(i + 6) | 32) == 'e' - ) { - return true; - } - } - return false; - } - - /** - * Validates the Sec-WebSocket-Key header. - * The key must be a base64-encoded 16-byte value. - * - * @param key the Sec-WebSocket-Key header value - * @return true if the key is valid - */ - public static boolean isValidKey(Utf8Sequence key) { - if (key == null) { - return false; - } - // Base64-encoded 16-byte value should be exactly 24 characters - // (16 bytes = 128 bits = 22 base64 chars + 2 padding = 24) - int size = key.size(); - if (size != 24) { - return false; - } - // Basic validation: check that all characters are valid base64 - for (int i = 0; i < size; i++) { - byte b = key.byteAt(i); - boolean valid = (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || - (b >= '0' && b <= '9') || b == '+' || b == '/' || b == '='; - if (!valid) { - return false; - } - } - return true; - } - - /** - * Validates the WebSocket version. - * - * @param versionHeader the Sec-WebSocket-Version header value - * @return true if the version is valid (13) - */ - public static boolean isValidVersion(Utf8Sequence versionHeader) { - if (versionHeader == null || versionHeader.size() == 0) { - return false; - } - // Parse the version number - try { - int version = 0; - for (int i = 0; i < versionHeader.size(); i++) { - byte b = versionHeader.byteAt(i); - if (b < '0' || b > '9') { - return false; - } - version = version * 10 + (b - '0'); - } - return version == WEBSOCKET_VERSION; - } catch (Exception e) { - return false; - } - } - - /** - * Checks if the given header indicates a WebSocket upgrade request. - * - * @param upgradeHeader the value of the Upgrade header - * @return true if this is a WebSocket upgrade request - */ - public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { - return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); - } - - /** - * Validates all required headers for a WebSocket upgrade request. - * - * @param upgradeHeader the Upgrade header value - * @param connectionHeader the Connection header value - * @param keyHeader the Sec-WebSocket-Key header value - * @param versionHeader the Sec-WebSocket-Version header value - * @return null if valid, or an error message describing the problem - */ - public static String validate( - Utf8Sequence upgradeHeader, - Utf8Sequence connectionHeader, - Utf8Sequence keyHeader, - Utf8Sequence versionHeader - ) { - if (!isWebSocketUpgrade(upgradeHeader)) { - return "Missing or invalid Upgrade header"; - } - if (!isConnectionUpgrade(connectionHeader)) { - return "Missing or invalid Connection header"; - } - if (!isValidKey(keyHeader)) { - return "Missing or invalid Sec-WebSocket-Key header"; - } - if (!isValidVersion(versionHeader)) { - return "Unsupported WebSocket version"; - } - return null; - } - -} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java index 31f1cc8..8668644 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -104,21 +104,4 @@ public static boolean isValid(int opcode) { || opcode == PONG; } - /** - * Returns a human-readable name for the opcode. - * - * @param opcode the opcode - * @return the opcode name - */ - public static String name(int opcode) { - return switch (opcode) { - case CONTINUATION -> "CONTINUATION"; - case TEXT -> "TEXT"; - case BINARY -> "BINARY"; - case CLOSE -> "CLOSE"; - case PING -> "PING"; - case PONG -> "PONG"; - default -> "UNKNOWN(" + opcode + ")"; - }; - } } From 159ee3d2a34396eb564295171166f8990d910292 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 15:23:52 +0100 Subject: [PATCH 133/230] Enhanced switch --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 64 +++++++------------ 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 783b7ea..12630b7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -778,69 +778,49 @@ public void addNull() { if (nullable) { ensureNullCapacity(size + 1); markNull(size); - size++; } else { // For non-nullable columns, store a sentinel/default value switch (type) { - case TYPE_BOOLEAN, TYPE_BYTE: - dataBuffer.putByte((byte) 0); - break; - case TYPE_SHORT, TYPE_CHAR: - dataBuffer.putShort((short) 0); - break; - case TYPE_INT: - dataBuffer.putInt(0); - break; - case TYPE_GEOHASH: - dataBuffer.putLong(-1L); - break; - case TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE: - dataBuffer.putLong(Long.MIN_VALUE); - break; - case TYPE_FLOAT: - dataBuffer.putFloat(Float.NaN); - break; - case TYPE_DOUBLE: - dataBuffer.putDouble(Double.NaN); - break; - case TYPE_STRING, TYPE_VARCHAR: - stringOffsets.putInt((int) stringData.getAppendOffset()); - break; - case TYPE_SYMBOL: - dataBuffer.putInt(-1); - break; - case TYPE_UUID: + case TYPE_BOOLEAN, TYPE_BYTE -> dataBuffer.putByte((byte) 0); + case TYPE_SHORT, TYPE_CHAR -> dataBuffer.putShort((short) 0); + case TYPE_INT -> dataBuffer.putInt(0); + case TYPE_GEOHASH -> dataBuffer.putLong(-1L); + case TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE -> + dataBuffer.putLong(Long.MIN_VALUE); + case TYPE_FLOAT -> dataBuffer.putFloat(Float.NaN); + case TYPE_DOUBLE -> dataBuffer.putDouble(Double.NaN); + case TYPE_STRING, TYPE_VARCHAR -> stringOffsets.putInt((int) stringData.getAppendOffset()); + case TYPE_SYMBOL -> dataBuffer.putInt(-1); + case TYPE_UUID -> { dataBuffer.putLong(Long.MIN_VALUE); dataBuffer.putLong(Long.MIN_VALUE); - break; - case TYPE_LONG256: + } + case TYPE_LONG256 -> { dataBuffer.putLong(Long.MIN_VALUE); dataBuffer.putLong(Long.MIN_VALUE); dataBuffer.putLong(Long.MIN_VALUE); dataBuffer.putLong(Long.MIN_VALUE); - break; - case TYPE_DECIMAL64: - dataBuffer.putLong(Decimals.DECIMAL64_NULL); - break; - case TYPE_DECIMAL128: + } + case TYPE_DECIMAL64 -> dataBuffer.putLong(Decimals.DECIMAL64_NULL); + case TYPE_DECIMAL128 -> { dataBuffer.putLong(Decimals.DECIMAL128_HI_NULL); dataBuffer.putLong(Decimals.DECIMAL128_LO_NULL); - break; - case TYPE_DECIMAL256: + } + case TYPE_DECIMAL256 -> { dataBuffer.putLong(Decimals.DECIMAL256_HH_NULL); dataBuffer.putLong(Decimals.DECIMAL256_HL_NULL); dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); - break; - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY: + } + case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> { ensureArrayCapacity(1, 0); arrayDims[valueCount] = 1; arrayShapes[arrayShapeOffset++] = 0; - break; + } } valueCount++; - size++; } + size++; } public void addShort(short value) { From 73e8986fe6289a10872f80229b6f1c1d55871458 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 15:24:04 +0100 Subject: [PATCH 134/230] Delete unused code --- .../cutlass/qwp/client/MicrobatchBuffer.java | 8 -- .../qwp/client/QwpWebSocketEncoder.java | 16 --- .../qwp/client/QwpWebSocketSender.java | 103 +----------------- .../cutlass/qwp/protocol/QwpColumnDef.java | 16 --- .../cutlass/qwp/protocol/QwpSchemaHash.java | 39 ------- .../cutlass/qwp/protocol/QwpTableBuffer.java | 7 -- 6 files changed, 1 insertion(+), 188 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 94cb89e..4f5bfbf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -220,14 +220,6 @@ public long getBufferPtr() { return bufferPtr; } - /** - * Returns the maximum symbol ID used in this batch. - * Used for delta symbol dictionary tracking. - */ - public int getMaxSymbolId() { - return maxSymbolId; - } - /** * Returns the number of rows in this buffer. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 34f3659..1177add 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -113,26 +113,10 @@ public QwpBufferWriter getBuffer() { return buffer; } - public boolean isDeltaSymbolDictEnabled() { - return (flags & FLAG_DELTA_SYMBOL_DICT) != 0; - } - public boolean isGorillaEnabled() { return (flags & FLAG_GORILLA) != 0; } - public void reset() { - buffer.reset(); - } - - public void setDeltaSymbolDictEnabled(boolean enabled) { - if (enabled) { - flags |= FLAG_DELTA_SYMBOL_DICT; - } else { - flags &= ~FLAG_DELTA_SYMBOL_DICT; - } - } - public void setGorillaEnabled(boolean enabled) { if (enabled) { flags |= FLAG_GORILLA; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 723c422..fe30160 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -311,28 +311,6 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS); } - /** - * Factory method for SenderBuilder integration. - */ - public static QwpWebSocketSender create( - String host, - int port, - boolean tlsEnabled, - int bufferSize, - String authToken, - String username, - String password - ) { - QwpWebSocketSender sender = new QwpWebSocketSender( - host, port, tlsEnabled, bufferSize, - 0, 0, 0, - 1 // window=1 for sync behavior - ); - // TODO: Store auth credentials for connection - sender.ensureConnected(); - return sender; - } - /** * Creates a sender without connecting. For testing only. *

    @@ -561,7 +539,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { @Override public Sender decimalColumn(CharSequence name, CharSequence value) { - if (value == null || value.length() == 0) return this; + if (value == null || value.isEmpty()) return this; checkNotClosed(); checkTableSelected(); try { @@ -686,22 +664,6 @@ public int getAutoFlushRows() { return autoFlushRows; } - /** - * Returns the global symbol dictionary. - * For testing and encoder integration. - */ - public GlobalSymbolDictionary getGlobalSymbolDictionary() { - return globalSymbolDictionary; - } - - /** - * Returns the in-flight window size. - * Window=1 means sync mode, window>1 means async mode. - */ - public int getInFlightWindowSize() { - return inFlightWindowSize; - } - /** * Returns the max symbol ID sent to the server. * Once sent over TCP, server is guaranteed to receive it (or connection dies). @@ -710,18 +672,6 @@ public int getMaxSentSymbolId() { return maxSentSymbolId; } - /** - * Registers a symbol in the global dictionary and returns its ID. - * For use with fast-path column buffer access. - */ - public int getOrAddGlobalSymbol(String value) { - int globalId = globalSymbolDictionary.getOrAddSymbol(value); - if (globalId > currentBatchMaxSymbolId) { - currentBatchMaxSymbolId = globalId; - } - return globalId; - } - /** * Returns the number of pending rows not yet flushed. * For testing. @@ -745,28 +695,6 @@ public QwpTableBuffer getTableBuffer(String tableName) { return buffer; } - /** - * Increments the pending row count for auto-flush tracking. - * Call this after adding a complete row via fast-path API. - * Triggers auto-flush if any threshold is exceeded. - */ - public void incrementPendingRowCount() { - if (pendingRowCount == 0) { - firstPendingRowTimeNanos = System.nanoTime(); - } - pendingRowCount++; - - // Check if any flush threshold is exceeded (same as sendRow()) - if (shouldAutoFlush()) { - if (inFlightWindowSize > 1) { - flushPendingRows(); - } else { - // Sync mode (window=1): flush directly with ACK wait - flushSync(); - } - } - } - /** * Adds an INT column value to the current row. @@ -783,13 +711,6 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { return this; } - /** - * Returns whether async mode is enabled (window size > 1). - */ - public boolean isAsyncMode() { - return inFlightWindowSize > 1; - } - /** * Returns whether Gorilla encoding is enabled. */ @@ -997,28 +918,6 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) return this; } - /** - * Adds encoded data to the active microbatch buffer. - * Triggers seal and swap if buffer is full. - */ - private void addToMicrobatch(long dataPtr, int length) { - // Ensure activeBuffer is ready for writing - ensureActiveBufferReady(); - - // If current buffer can't hold the data, seal and swap - if (activeBuffer.hasData() && - (long) activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { - sealAndSwapBuffer(); - } - - // Ensure buffer can hold the data - activeBuffer.ensureCapacity((int) Math.min((long) activeBuffer.getBufferPos() + length, Integer.MAX_VALUE)); - - // Copy data to buffer - activeBuffer.write(dataPtr, length); - activeBuffer.incrementRowCount(); - } - private void atMicros(long timestampMicros) { // Add designated timestamp column (empty name for designated timestamp) // Use cached reference to avoid hashmap lookup per row diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index 8c2c65a..a2f4f50 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -73,15 +73,6 @@ public boolean equals(Object o) { name.equals(that.name); } - /** - * Gets the fixed width in bytes for fixed-width types. - * - * @return width in bytes, or -1 for variable-width types - */ - public int getFixedWidth() { - return QwpConstants.getFixedTypeSize(typeCode); - } - /** * Gets the column name. */ @@ -122,13 +113,6 @@ public int hashCode() { return result; } - /** - * Returns true if this is a fixed-width type. - */ - public boolean isFixedWidth() { - return QwpConstants.isFixedWidthType(typeCode); - } - /** * Returns true if this column is nullable. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 07f5c38..63d005d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -26,7 +26,6 @@ import io.questdb.client.std.Unsafe; -import io.questdb.client.std.str.Utf8Sequence; /** * XXHash64 implementation for schema hashing in ILP v4 protocol. @@ -57,32 +56,6 @@ private QwpSchemaHash() { // utility class } - /** - * Computes the schema hash for ILP v4. - *

    - * Hash is computed over: for each column, hash(name_bytes + type_byte) - * This matches the spec in Appendix C. - * - * @param columnNames array of column names (UTF-8) - * @param columnTypes array of type codes - * @return the schema hash - */ - public static long computeSchemaHash(Utf8Sequence[] columnNames, byte[] columnTypes) { - // Use pooled hasher to avoid allocation - Hasher hasher = HASHER_POOL.get(); - hasher.reset(DEFAULT_SEED); - - for (int i = 0; i < columnNames.length; i++) { - Utf8Sequence name = columnNames[i]; - for (int j = 0, n = name.size(); j < n; j++) { - hasher.update(name.byteAt(j)); - } - hasher.update(columnTypes[i]); - } - - return hasher.getValue(); - } - /** * Computes the schema hash for ILP v4 using String column names. * Note: Iterates over String chars and converts to UTF-8 bytes directly to avoid getBytes() allocation. @@ -268,18 +241,6 @@ public static long hash(byte[] data) { return hash(data, 0, data.length, DEFAULT_SEED); } - /** - * Computes XXHash64 of a byte array region. - * - * @param data the data to hash - * @param offset starting offset - * @param length number of bytes to hash - * @return the 64-bit hash value - */ - public static long hash(byte[] data, int offset, int length) { - return hash(data, offset, length, DEFAULT_SEED); - } - /** * Computes XXHash64 of a byte array region with custom seed. * diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 12630b7..52d9efd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -946,13 +946,6 @@ public long getDataAddress() { return dataBuffer != null ? dataBuffer.pageAddress() : 0; } - /** - * Returns the number of bytes of data in the off-heap buffer. - */ - public long getDataSize() { - return dataBuffer != null ? dataBuffer.getAppendOffset() : 0; - } - public byte getDecimalScale() { return decimalScale; } From eceded129b3a94609a8515af39261d698573e6ff Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 15:29:49 +0100 Subject: [PATCH 135/230] Code style --- .../cutlass/qwp/protocol/QwpBitWriter.java | 2 +- .../java/io/questdb/client/network/Net.java | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index 5e217f1..fbe6efd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -158,7 +158,7 @@ public void writeBits(long value, int numBits) { int bitsToWrite = numBits; while (bitsToWrite > 0) { - // How many bits can we fit in current buffer (max 64 total) + // How many bits can fit into the current buffer (max 64 total) int availableInBuffer = 64 - bitsInBuffer; int bitsThisRound = Math.min(bitsToWrite, availableInBuffer); diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index fe21505..b1c4721 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -80,9 +80,9 @@ public static void configureKeepAlive(int fd) { public static native int configureNonBlocking(int fd); - public native static int connect(int fd, long sockaddr); + public static native int connect(int fd, long sockaddr); - public native static int connectAddrInfo(int fd, long lpAddrInfo); + public static native int connectAddrInfo(int fd, long lpAddrInfo); public static void freeAddrInfo(long pAddrInfo) { if (pAddrInfo != 0) { @@ -106,13 +106,13 @@ public static long getAddrInfo(CharSequence host, int port) { } } - public native static int getSndBuf(int fd); + public static native int getSndBuf(int fd); public static void init() { // no-op } - public native static boolean join(int fd, int bindIPv4Address, int groupIPv4Address); + public static native boolean join(int fd, int bindIPv4Address, int groupIPv4Address); public static native int peek(int fd, long ptr, int len); @@ -120,28 +120,28 @@ public static void init() { public static native int send(int fd, long ptr, int len); - public native static int sendTo(int fd, long ptr, int len, long sockaddr); + public static native int sendTo(int fd, long ptr, int len, long sockaddr); public static native int setKeepAlive0(int fd, int seconds); - public native static int setMulticastInterface(int fd, int ipv4address); + public static native int setMulticastInterface(int fd, int ipv4address); - public native static int setMulticastTtl(int fd, int ttl); + public static native int setMulticastTtl(int fd, int ttl); - public native static int setSndBuf(int fd, int size); + public static native int setSndBuf(int fd, int size); - public native static int setTcpNoDelay(int fd, boolean noDelay); + public static native int setTcpNoDelay(int fd, boolean noDelay); public static long sockaddr(int ipv4address, int port) { SOCK_ADDR_COUNTER.incrementAndGet(); return sockaddr0(ipv4address, port); } - public native static long sockaddr0(int ipv4address, int port); + public static native long sockaddr0(int ipv4address, int port); - public native static int socketTcp(boolean blocking); + public static native int socketTcp(boolean blocking); - public native static int socketUdp(); + public static native int socketUdp(); private static native void freeAddrInfo0(long pAddrInfo); From 054c532bca31c19f76662da6844c9bfaa993be22 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 15:40:29 +0100 Subject: [PATCH 136/230] Cache Maven dependencies in CI pipeline Add a Cache@2 task to the Azure Pipelines BuildAndTest matrix so that the 4 platform jobs (linux, mac-arm, mac-x64, windows) reuse downloaded Maven dependencies across runs instead of fetching them from scratch each time. The cache key incorporates the OS and all pom.xml hashes. A restoreKeys fallback on just the OS ensures partial cache hits when dependencies change. Co-Authored-By: Claude Opus 4.6 --- ci/run_tests_pipeline.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index a430af4..53a3305 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -72,6 +72,13 @@ stages: lfs: false submodules: false - template: setup.yaml + - task: Cache@2 + inputs: + key: 'maven | "$(Agent.OS)" | **/pom.xml' + restoreKeys: | + maven | "$(Agent.OS)" + path: $(HOME)/.m2/repository + displayName: "Cache Maven repository" # TODO: remove branch once jh_experiment_new_ilp is merged - script: git clone --depth 1 -b jh_experiment_new_ilp https://github.com/questdb/questdb.git ./questdb displayName: git clone questdb From 04bf6c43a122c6307b3360447f3421e7bacbb97a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 15:44:32 +0100 Subject: [PATCH 137/230] Ensure QuestDB cleanup runs on test failure Add always() to the condition expressions in questdb_stop.yaml so the stop steps execute regardless of whether prior steps succeeded. Without this, Azure DevOps defaults to succeeded() and skips the cleanup when Maven tests fail, leaving the QuestDB server process orphaned until the VM is torn down. Co-Authored-By: Claude Opus 4.6 --- ci/questdb_stop.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/questdb_stop.yaml b/ci/questdb_stop.yaml index 3b6c0d9..5b56935 100644 --- a/ci/questdb_stop.yaml +++ b/ci/questdb_stop.yaml @@ -5,7 +5,7 @@ steps: rm -f questdb.pid rm -rf questdb-data displayName: "Stop QuestDB server (non-Windows)" - condition: ne(variables['Agent.OS'], 'Windows_NT') + condition: and(always(), ne(variables['Agent.OS'], 'Windows_NT')) - pwsh: | Write-Host "Stopping QuestDB server..." if (Test-Path "questdb.pid") { @@ -36,4 +36,4 @@ steps: } } displayName: "Stop QuestDB server (Windows)" - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(always(), eq(variables['Agent.OS'], 'Windows_NT')) From 5df814090b5c12158c377f571c75d828561c157d Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 3 Mar 2026 16:30:25 +0100 Subject: [PATCH 138/230] Fix Maven cache on Windows CI agents The Cache@2 task uses $(HOME)/.m2/repository as the cache path. On Linux/macOS agents, HOME is a native environment variable that Azure DevOps imports as a pipeline variable. On Windows, HOME is not set natively (Windows uses USERPROFILE), so $(HOME) remains unexpanded as a literal string, causing tar to fail with "could not chdir to 'D:\a\1\s\$(HOME)\.m2\repository'". Add a step before the cache task that sets HOME from USERPROFILE on Windows agents via ##vso[task.setvariable]. The step runs only when Agent.OS is Windows_NT, leaving Linux/macOS unchanged. Co-Authored-By: Claude Opus 4.6 --- ci/run_tests_pipeline.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 53a3305..00c0f8c 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -72,6 +72,9 @@ stages: lfs: false submodules: false - template: setup.yaml + - bash: echo "##vso[task.setvariable variable=HOME]$USERPROFILE" + displayName: "Set HOME on Windows" + condition: eq(variables['Agent.OS'], 'Windows_NT') - task: Cache@2 inputs: key: 'maven | "$(Agent.OS)" | **/pom.xml' From f401d8e2abbe04319b05ef50123d5698772f9f96 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 4 Mar 2026 10:00:04 +0100 Subject: [PATCH 139/230] wip: udp sender --- .../cutlass/qwp/client/QwpUdpSender.java | 425 ++++++++++++++++++ 1 file changed, 425 insertions(+) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java new file mode 100644 index 0000000..d1aa5df --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -0,0 +1,425 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.line.udp.UdpLineChannel; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.CharSequenceObjHashMap; +import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.bytes.DirectByteSlice; +import io.questdb.client.network.NetworkFacade; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Fire-and-forget ILP v4 sender over UDP. + *

    + * Each {@link #flush()} encodes all buffered table data into self-contained + * datagrams (one per table) and sends them via UDP. Datagrams use local + * symbol dictionaries (no global/delta dict) and full schema (no schema refs). + */ +public class QwpUdpSender implements Sender { + private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); + + private final UdpLineChannel channel; + private final QwpWebSocketEncoder encoder; + private final CharSequenceObjHashMap tableBuffers; + + private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; + private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; + private boolean closed; + private QwpTableBuffer currentTableBuffer; + private String currentTableName; + + public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { + this.encoder = new QwpWebSocketEncoder(); + this.encoder.setGorillaEnabled(false); + this.channel = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); + this.tableBuffers = new CharSequenceObjHashMap<>(); + } + + @Override + public void at(long timestamp, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + atNanos(timestamp); + } else { + long micros = toMicros(timestamp, unit); + atMicros(micros); + } + } + + @Override + public void at(Instant timestamp) { + checkNotClosed(); + checkTableSelected(); + long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; + atMicros(micros); + } + + @Override + public void atNow() { + checkNotClosed(); + checkTableSelected(); + currentTableBuffer.nextRow(); + } + + @Override + public Sender boolColumn(CharSequence columnName, boolean value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + col.addBoolean(value); + return this; + } + + @Override + public DirectByteSlice bufferView() { + throw new LineSenderException("bufferView() is not supported for UDP sender"); + } + + @Override + public void cancelRow() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + } + } + + @Override + public void close() { + if (!closed) { + try { + flushInternal(); + } catch (Exception e) { + LOG.error("Error during close flush: {}", String.valueOf(e)); + } + closed = true; + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + tb.close(); + } + } + } + tableBuffers.clear(); + channel.close(); + encoder.close(); + } + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal64 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + col.addDecimal64(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal128 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + col.addDecimal128(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal256 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(value); + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(CharSequence name, DoubleArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(array); + return this; + } + + @Override + public Sender doubleColumn(CharSequence columnName, double value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); + col.addDouble(value); + return this; + } + + @Override + public void flush() { + checkNotClosed(); + flushInternal(); + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, LongArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(array); + return this; + } + + @Override + public Sender longColumn(CharSequence columnName, long value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + col.addLong(value); + return this; + } + + @Override + public void reset() { + checkNotClosed(); + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.reset(); + } + } + currentTableBuffer = null; + currentTableName = null; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + + @Override + public Sender stringColumn(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + col.addString(value != null ? value.toString() : null); + return this; + } + + @Override + public Sender symbol(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + if (value != null) { + col.addSymbol(value.toString()); + } else { + col.addSymbol(null); + } + return this; + } + + @Override + public Sender table(CharSequence tableName) { + checkNotClosed(); + if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { + return this; + } + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableName = tableName.toString(); + currentTableBuffer = tableBuffers.get(currentTableName); + if (currentTableBuffer == null) { + currentTableBuffer = new QwpTableBuffer(currentTableName); + tableBuffers.put(currentTableName, currentTableBuffer); + } + return this; + } + + @Override + public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); + col.addLong(value); + } else { + long micros = toMicros(value, unit); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + col.addLong(micros); + } + return this; + } + + @Override + public Sender timestampColumn(CharSequence columnName, Instant value) { + checkNotClosed(); + checkTableSelected(); + long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + col.addLong(micros); + return this; + } + + private void atMicros(long timestampMicros) { + if (cachedTimestampColumn == null) { + cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + } + cachedTimestampColumn.addLong(timestampMicros); + currentTableBuffer.nextRow(); + } + + private void atNanos(long timestampNanos) { + if (cachedTimestampNanosColumn == null) { + cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); + } + cachedTimestampNanosColumn.addLong(timestampNanos); + currentTableBuffer.nextRow(); + } + + private void checkNotClosed() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + } + + private void checkTableSelected() { + if (currentTableBuffer == null) { + throw new LineSenderException("table() must be called before adding columns"); + } + } + + private void flushInternal() { + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) continue; + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) continue; + + int len = encoder.encode(tableBuffer, false); + try { + channel.send(encoder.getBuffer().getBufferPtr(), len); + } catch (LineSenderException e) { + LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); + } + tableBuffer.reset(); + } + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + + private long toMicros(long value, ChronoUnit unit) { + return switch (unit) { + case NANOS -> value / 1000L; + case MICROS -> value; + case MILLIS -> value * 1000L; + case SECONDS -> value * 1_000_000L; + case MINUTES -> value * 60_000_000L; + case HOURS -> value * 3_600_000_000L; + case DAYS -> value * 86_400_000_000L; + default -> throw new LineSenderException("Unsupported time unit: " + unit); + }; + } +} From c9a0d04d9f62f9737679372ff0bb0277bd4f0bfe Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 4 Mar 2026 14:47:54 +0100 Subject: [PATCH 140/230] Add byte-based auto-flush to QWP sender QwpWebSocketSender.shouldAutoFlush() accepted an autoFlushBytes parameter but never evaluated it. This commit implements the byte threshold by querying actual column buffer sizes rather than maintaining an estimated counter. This counts the bytes in column buffers, before wire-encoding. Encoded size will be less due to Gorilla compression, bit-packing, etc. QwpTableBuffer.getBufferedBytes() sums OffHeapAppendMemory append offsets and array data across all columns. QwpWebSocketSender.getPendingBytes() aggregates this across all table buffers. shouldAutoFlush() now checks this sum against autoFlushBytes between the row-count and interval checks. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 21 ++++++++-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 38 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index fe30160..3b77d73 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1192,6 +1192,21 @@ private void flushSync() { LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); } + private long getPendingBytes() { + long bytes = 0; + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + bytes += tb.getBufferedBytes(); + } + } + } + return bytes; + } + /** * Seals the current buffer and swaps to the other buffer. * Enqueues the sealed buffer for async sending. @@ -1273,16 +1288,16 @@ private boolean shouldAutoFlush() { if (pendingRowCount <= 0) { return false; } - // Row limit if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { return true; } - // Time limit + if (autoFlushBytes > 0 && getPendingBytes() >= autoFlushBytes) { + return true; + } if (autoFlushIntervalNanos > 0) { long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; return ageNanos >= autoFlushIntervalNanos; } - // Byte limit is harder to estimate without encoding, skip for now return false; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 52d9efd..f3d7fe1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -113,6 +113,18 @@ public void close() { clear(); } + /** + * Returns the total bytes buffered across all columns. + * This queries actual buffer sizes, not estimates. + */ + public long getBufferedBytes() { + long bytes = 0; + for (int i = 0, n = columns.size(); i < n; i++) { + bytes += fastColumns[i].getBufferedBytes(); + } + return bytes; + } + /** * Returns the column at the given index. */ @@ -931,6 +943,32 @@ public int[] getArrayShapes() { return arrayShapes; } + /** + * Returns the total bytes buffered in this column's storage. + */ + public long getBufferedBytes() { + long bytes = 0; + if (dataBuffer != null) { + bytes += dataBuffer.getAppendOffset(); + } + if (auxBuffer != null) { + bytes += auxBuffer.getAppendOffset(); + } + if (stringData != null) { + bytes += stringData.getAppendOffset(); + } + if (stringOffsets != null) { + bytes += stringOffsets.getAppendOffset(); + } + if (doubleArrayData != null) { + bytes += (long) arrayDataOffset * Double.BYTES; + } + if (longArrayData != null) { + bytes += (long) arrayDataOffset * Long.BYTES; + } + return bytes; + } + /** * Returns the off-heap address of the auxiliary data buffer (global symbol IDs). * Returns 0 if no auxiliary data exists. From 3457d8eacba3ef822be3c98d1b79f0804e273e55 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 4 Mar 2026 15:19:04 +0100 Subject: [PATCH 141/230] iteration #3 --- .../qwp/client/QwpDatagramSizeEstimator.java | 215 +++++++ .../cutlass/qwp/client/QwpUdpSender.java | 348 ++++++++++- .../client/QwpDatagramSizeEstimatorTest.java | 564 ++++++++++++++++++ 3 files changed, 1103 insertions(+), 24 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java new file mode 100644 index 0000000..4d5ec39 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java @@ -0,0 +1,215 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Estimates the encoded datagram size for a {@link QwpTableBuffer}. + *

    + * The estimate mirrors the encoding in {@link QwpWebSocketEncoder#encode} + * with {@code useSchemaRef=false} and Gorilla disabled. The estimate is + * always >= the actual encoded size. + */ +public final class QwpDatagramSizeEstimator { + + private QwpDatagramSizeEstimator() { + } + + /** + * Estimates the encoded datagram size in bytes. + *

    + * The {@code rowCount} parameter is separate from {@code tableBuffer.getRowCount()} + * so the caller can pass {@code rowCount + 1} for a tentative pre-commit estimate. + * + * @param tableBuffer the table buffer to estimate + * @param rowCount the number of rows to estimate for + * @return the estimated encoded size in bytes (always >= actual) + */ + public static long estimate(QwpTableBuffer tableBuffer, int rowCount) { + long size = 0; + + // Header: ILP4 (4) + version (1) + flags (1) + tableCount (2) + payloadLength (4) = 12 + size += HEADER_SIZE; + + // Table name + String tableName = tableBuffer.getTableName(); + int tableNameUtf8Len = NativeBufferWriter.utf8Length(tableName); + size += varintSize(tableNameUtf8Len) + tableNameUtf8Len; + + // Row count varint + size += varintSize(rowCount); + + int columnCount = tableBuffer.getColumnCount(); + // Column count varint + size += varintSize(columnCount); + + // Schema mode byte + size += 1; + + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + + // Per-column schema: name + wire type code + for (int i = 0; i < columnCount; i++) { + QwpColumnDef colDef = columnDefs[i]; + int nameUtf8Len = NativeBufferWriter.utf8Length(colDef.getName()); + size += varintSize(nameUtf8Len) + nameUtf8Len; + size += 1; // wire type code + } + + // Per-column data + for (int i = 0; i < columnCount; i++) { + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + QwpColumnDef colDef = columnDefs[i]; + int valueCount = col.getValueCount(); + + // Nullable bitmap + if (colDef.isNullable()) { + size += (rowCount + 7) / 8; + } + + // Safety margin: if this column has fewer values than rows, + // nextRow() will pad it -- account for one extra element + if (col.getSize() < rowCount) { + size += elementWireSize(col.getType()); + } + + size += estimateColumnData(col, valueCount); + } + + // Fixed safety margin + size += 8; + + return size; + } + + public static int varintSize(long value) { + if (value == 0) { + return 1; + } + return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; + } + + private static long estimateArrayColumn(QwpTableBuffer.ColumnBuffer col, int valueCount, int elemBytes) { + long size = 0; + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + if (dims == null || shapes == null) { + return size; + } + + int shapeIdx = 0; + for (int row = 0; row < valueCount; row++) { + int nDims = dims[row]; + // nDims byte + size += 1; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + // dim length int32 + size += 4; + elemCount *= shapes[shapeIdx++]; + } + // elements + size += (long) elemCount * elemBytes; + } + return size; + } + + private static long estimateColumnData(QwpTableBuffer.ColumnBuffer col, int valueCount) { + return switch (col.getType()) { + case TYPE_BOOLEAN -> (valueCount + 7) / 8; + case TYPE_BYTE -> valueCount; + case TYPE_SHORT, TYPE_CHAR -> (long) valueCount * 2; + case TYPE_INT, TYPE_FLOAT -> (long) valueCount * 4; + case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE -> (long) valueCount * 8; + case TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> + // Gorilla is disabled for UDP, so no encoding byte -- just raw longs + (long) valueCount * 8; + case TYPE_UUID -> (long) valueCount * 16; + case TYPE_LONG256 -> (long) valueCount * 32; + case TYPE_DECIMAL64 -> 1 + (long) valueCount * 8; + case TYPE_DECIMAL128 -> 1 + (long) valueCount * 16; + case TYPE_DECIMAL256 -> 1 + (long) valueCount * 32; + case TYPE_STRING, TYPE_VARCHAR -> + (long) (valueCount + 1) * 4 + col.getStringDataSize(); + case TYPE_SYMBOL -> estimateSymbolColumn(col, valueCount); + case TYPE_GEOHASH -> estimateGeoHashColumn(col, valueCount); + case TYPE_DOUBLE_ARRAY -> estimateArrayColumn(col, valueCount, 8); + case TYPE_LONG_ARRAY -> estimateArrayColumn(col, valueCount, 8); + default -> 0; + }; + } + + private static long estimateGeoHashColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) { + int precision = col.getGeoHashPrecision(); + if (precision < 1) { + precision = 1; + } + long size = varintSize(precision); + int valueSize = (precision + 7) / 8; + size += (long) valueCount * valueSize; + return size; + } + + private static long estimateSymbolColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) { + String[] dictionary = col.getSymbolDictionary(); + int dictSize = dictionary.length; + + long size = varintSize(dictSize); + for (String symbol : dictionary) { + int utf8Len = NativeBufferWriter.utf8Length(symbol); + size += varintSize(utf8Len) + utf8Len; + } + + // Per-value index varints. Maximum index is dictSize - 1. + int maxIndex = Math.max(0, dictSize - 1); + size += (long) valueCount * varintSize(maxIndex); + + return size; + } + + private static int elementWireSize(byte type) { + return switch (type) { + case TYPE_BOOLEAN -> 1; + case TYPE_BYTE -> 1; + case TYPE_SHORT, TYPE_CHAR -> 2; + case TYPE_INT, TYPE_FLOAT -> 4; + case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> 8; + case TYPE_UUID -> 16; + case TYPE_LONG256 -> 32; + case TYPE_DECIMAL64 -> 8; + case TYPE_DECIMAL128 -> 16; + case TYPE_DECIMAL256 -> 32; + case TYPE_STRING, TYPE_VARCHAR -> 4; // one offset entry + case TYPE_SYMBOL -> 1; // one varint index (at least 1 byte) + case TYPE_GEOHASH -> 8; + case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> 5; // 1 dim byte + 4 shape int + default -> 0; + }; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index d1aa5df..5255daa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -53,12 +53,32 @@ * Each {@link #flush()} encodes all buffered table data into self-contained * datagrams (one per table) and sends them via UDP. Datagrams use local * symbol dictionaries (no global/delta dict) and full schema (no schema refs). + *

    + * When {@code maxDatagramSize > 0}, the sender automatically flushes before + * a datagram exceeds the size limit. The current in-progress row is cancelled, + * committed rows are flushed, and the in-progress row is replayed from a journal. */ public class QwpUdpSender implements Sender { + private static final byte ENTRY_AT_MICROS = 1; + private static final byte ENTRY_AT_NANOS = 2; + private static final byte ENTRY_BOOL = 3; + private static final byte ENTRY_DECIMAL128 = 4; + private static final byte ENTRY_DECIMAL256 = 5; + private static final byte ENTRY_DECIMAL64 = 6; + private static final byte ENTRY_DOUBLE = 7; + private static final byte ENTRY_DOUBLE_ARRAY = 8; + private static final byte ENTRY_LONG = 9; + private static final byte ENTRY_LONG_ARRAY = 10; + private static final byte ENTRY_STRING = 11; + private static final byte ENTRY_SYMBOL = 12; + private static final byte ENTRY_TIMESTAMP_COL_MICROS = 13; + private static final byte ENTRY_TIMESTAMP_COL_NANOS = 14; private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); private final UdpLineChannel channel; private final QwpWebSocketEncoder encoder; + private final int maxDatagramSize; + private final ObjList rowJournal = new ObjList<>(); private final CharSequenceObjHashMap tableBuffers; private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; @@ -66,12 +86,18 @@ public class QwpUdpSender implements Sender { private boolean closed; private QwpTableBuffer currentTableBuffer; private String currentTableName; + private int rowJournalSize; public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { + this(nf, interfaceIPv4, sendToAddress, port, ttl, 0); + } + + public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl, int maxDatagramSize) { this.encoder = new QwpWebSocketEncoder(); this.encoder.setGorillaEnabled(false); this.channel = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); this.tableBuffers = new CharSequenceObjHashMap<>(); + this.maxDatagramSize = maxDatagramSize; } @Override @@ -98,15 +124,26 @@ public void at(Instant timestamp) { public void atNow() { checkNotClosed(); checkTableSelected(); + if (maxDatagramSize > 0) { + maybeAutoFlush(); + } currentTableBuffer.nextRow(); + rowJournalSize = 0; } @Override public Sender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + String name = columnName.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_BOOLEAN, false); col.addBoolean(value); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_BOOL; + e.name = name; + e.boolValue = value; + } return this; } @@ -121,6 +158,7 @@ public void cancelRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); } + rowJournalSize = 0; } @Override @@ -153,8 +191,15 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DECIMAL64, true); col.addDecimal64(value); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DECIMAL64; + e.name = colName; + e.objectValue = value; + } return this; } @@ -163,8 +208,15 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DECIMAL128, true); col.addDecimal128(value); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DECIMAL128; + e.name = colName; + e.objectValue = value; + } return this; } @@ -173,8 +225,15 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DECIMAL256, true); col.addDecimal256(value); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DECIMAL256; + e.name = colName; + e.objectValue = value; + } return this; } @@ -183,8 +242,15 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DOUBLE_ARRAY; + e.name = colName; + e.objectValue = values; + } return this; } @@ -193,8 +259,15 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DOUBLE_ARRAY; + e.name = colName; + e.objectValue = values; + } return this; } @@ -203,8 +276,15 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DOUBLE_ARRAY; + e.name = colName; + e.objectValue = values; + } return this; } @@ -213,8 +293,15 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(array); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DOUBLE_ARRAY; + e.name = colName; + e.objectValue = array; + } return this; } @@ -222,8 +309,15 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public Sender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); + String name = columnName.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE, false); col.addDouble(value); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DOUBLE; + e.name = name; + e.doubleValue = value; + } return this; } @@ -238,8 +332,15 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); col.addLongArray(values); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_LONG_ARRAY; + e.name = colName; + e.objectValue = values; + } return this; } @@ -248,8 +349,15 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); col.addLongArray(values); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_LONG_ARRAY; + e.name = colName; + e.objectValue = values; + } return this; } @@ -258,8 +366,15 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); col.addLongArray(values); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_LONG_ARRAY; + e.name = colName; + e.objectValue = values; + } return this; } @@ -268,8 +383,15 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + String colName = name.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); col.addLongArray(array); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_LONG_ARRAY; + e.name = colName; + e.objectValue = array; + } return this; } @@ -277,8 +399,15 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public Sender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + String name = columnName.toString(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG, false); col.addLong(value); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_LONG; + e.name = name; + e.longValue = value; + } return this; } @@ -296,14 +425,23 @@ public void reset() { currentTableName = null; cachedTimestampColumn = null; cachedTimestampNanosColumn = null; + rowJournalSize = 0; } @Override public Sender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); - col.addString(value != null ? value.toString() : null); + String name = columnName.toString(); + String strValue = value != null ? value.toString() : null; + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_STRING, true); + col.addString(strValue); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_STRING; + e.name = name; + e.stringValue = strValue; + } return this; } @@ -311,11 +449,15 @@ public Sender stringColumn(CharSequence columnName, CharSequence value) { public Sender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); - if (value != null) { - col.addSymbol(value.toString()); - } else { - col.addSymbol(null); + String name = columnName.toString(); + String strValue = value != null ? value.toString() : null; + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_SYMBOL, true); + col.addSymbol(strValue); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_SYMBOL; + e.name = name; + e.stringValue = strValue; } return this; } @@ -326,8 +468,13 @@ public Sender table(CharSequence tableName) { if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { return this; } + // Flush current table on switch if auto-flush is enabled and there are committed rows + if (maxDatagramSize > 0 && currentTableBuffer != null && currentTableBuffer.getRowCount() > 0) { + flushSingleTable(currentTableName, currentTableBuffer); + } cachedTimestampColumn = null; cachedTimestampNanosColumn = null; + rowJournalSize = 0; currentTableName = tableName.toString(); currentTableBuffer = tableBuffers.get(currentTableName); if (currentTableBuffer == null) { @@ -341,13 +488,26 @@ public Sender table(CharSequence tableName) { public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); + String name = columnName.toString(); if (unit == ChronoUnit.NANOS) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_TIMESTAMP_NANOS, true); col.addLong(value); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_TIMESTAMP_COL_NANOS; + e.name = name; + e.longValue = value; + } } else { long micros = toMicros(value, unit); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_TIMESTAMP, true); col.addLong(micros); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_TIMESTAMP_COL_MICROS; + e.name = name; + e.longValue = micros; + } } return this; } @@ -356,9 +516,16 @@ public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit un public Sender timestampColumn(CharSequence columnName, Instant value) { checkNotClosed(); checkTableSelected(); + String name = columnName.toString(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_TIMESTAMP, true); col.addLong(micros); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_TIMESTAMP_COL_MICROS; + e.name = name; + e.longValue = micros; + } return this; } @@ -367,7 +534,14 @@ private void atMicros(long timestampMicros) { cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); } cachedTimestampColumn.addLong(timestampMicros); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_AT_MICROS; + e.longValue = timestampMicros; + maybeAutoFlush(); + } currentTableBuffer.nextRow(); + rowJournalSize = 0; } private void atNanos(long timestampNanos) { @@ -375,7 +549,14 @@ private void atNanos(long timestampNanos) { cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); } cachedTimestampNanosColumn.addLong(timestampNanos); + if (maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_AT_NANOS; + e.longValue = timestampNanos; + maybeAutoFlush(); + } currentTableBuffer.nextRow(); + rowJournalSize = 0; } private void checkNotClosed() { @@ -410,6 +591,115 @@ private void flushInternal() { cachedTimestampNanosColumn = null; } + private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { + int len = encoder.encode(tableBuffer, false); + try { + channel.send(encoder.getBuffer().getBufferPtr(), len); + } catch (LineSenderException e) { + LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); + } + tableBuffer.reset(); + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + + private void maybeAutoFlush() { + int tentativeRowCount = currentTableBuffer.getRowCount() + 1; + long estimate = QwpDatagramSizeEstimator.estimate(currentTableBuffer, tentativeRowCount); + if (estimate > maxDatagramSize) { + if (currentTableBuffer.getRowCount() == 0) { + throw new LineSenderException( + "single row exceeds maximum datagram size (" + maxDatagramSize + + " bytes), estimated " + estimate + " bytes" + ); + } + currentTableBuffer.cancelCurrentRow(); + flushSingleTable(currentTableName, currentTableBuffer); + replayRowJournal(); + } + } + + private ColumnEntry nextJournalEntry() { + if (rowJournalSize < rowJournal.size()) { + ColumnEntry entry = rowJournal.getQuick(rowJournalSize); + entry.objectValue = null; + entry.stringValue = null; + rowJournalSize++; + return entry; + } + ColumnEntry entry = new ColumnEntry(); + rowJournal.add(entry); + rowJournalSize++; + return entry; + } + + private void replayDoubleArray(ColumnEntry entry) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DOUBLE_ARRAY, true); + if (entry.objectValue instanceof double[] a) { + col.addDoubleArray(a); + } else if (entry.objectValue instanceof double[][] a) { + col.addDoubleArray(a); + } else if (entry.objectValue instanceof double[][][] a) { + col.addDoubleArray(a); + } else if (entry.objectValue instanceof DoubleArray a) { + col.addDoubleArray(a); + } + } + + private void replayLongArray(ColumnEntry entry) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(entry.name, TYPE_LONG_ARRAY, true); + if (entry.objectValue instanceof long[] a) { + col.addLongArray(a); + } else if (entry.objectValue instanceof long[][] a) { + col.addLongArray(a); + } else if (entry.objectValue instanceof long[][][] a) { + col.addLongArray(a); + } else if (entry.objectValue instanceof LongArray a) { + col.addLongArray(a); + } + } + + private void replayRowJournal() { + for (int i = 0; i < rowJournalSize; i++) { + ColumnEntry entry = rowJournal.getQuick(i); + switch (entry.kind) { + case ENTRY_AT_MICROS -> { + cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + cachedTimestampColumn.addLong(entry.longValue); + } + case ENTRY_AT_NANOS -> { + cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); + cachedTimestampNanosColumn.addLong(entry.longValue); + } + case ENTRY_BOOL -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_BOOLEAN, false).addBoolean(entry.boolValue); + case ENTRY_DECIMAL128 -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DECIMAL128, true) + .addDecimal128((Decimal128) entry.objectValue); + case ENTRY_DECIMAL256 -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DECIMAL256, true) + .addDecimal256((Decimal256) entry.objectValue); + case ENTRY_DECIMAL64 -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DECIMAL64, true) + .addDecimal64((Decimal64) entry.objectValue); + case ENTRY_DOUBLE -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DOUBLE, false).addDouble(entry.doubleValue); + case ENTRY_DOUBLE_ARRAY -> replayDoubleArray(entry); + case ENTRY_LONG -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_LONG, false).addLong(entry.longValue); + case ENTRY_LONG_ARRAY -> replayLongArray(entry); + case ENTRY_STRING -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_STRING, true).addString(entry.stringValue); + case ENTRY_SYMBOL -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_SYMBOL, true).addSymbol(entry.stringValue); + case ENTRY_TIMESTAMP_COL_MICROS -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_TIMESTAMP, true).addLong(entry.longValue); + case ENTRY_TIMESTAMP_COL_NANOS -> + currentTableBuffer.getOrCreateColumn(entry.name, TYPE_TIMESTAMP_NANOS, true).addLong(entry.longValue); + } + } + } + private long toMicros(long value, ChronoUnit unit) { return switch (unit) { case NANOS -> value / 1000L; @@ -422,4 +712,14 @@ private long toMicros(long value, ChronoUnit unit) { default -> throw new LineSenderException("Unsupported time unit: " + unit); }; } + + private static class ColumnEntry { + boolean boolValue; + double doubleValue; + byte kind; + long longValue; + String name; + Object objectValue; + String stringValue; + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java new file mode 100644 index 0000000..fd080ba --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java @@ -0,0 +1,564 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpDatagramSizeEstimator; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Random; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpDatagramSizeEstimatorTest { + + @Test + public void testBooleanColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_BOOLEAN, false).addBoolean(true); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testByteColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_BYTE, false).addByte((byte) 42); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testCharColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_CHAR, false).addShort((short) 'A'); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDateColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DATE, false).addLong(1_700_000_000_000L); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDecimal128Column() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DECIMAL128, true).addDecimal128(new Decimal128(0, 12345, 2)); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDecimal256Column() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DECIMAL256, true).addDecimal256(new Decimal256(0, 0, 0, 12345, 2)); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDecimal64Column() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DECIMAL64, true).addDecimal64(new Decimal64(12345, 2)); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDoubleArray2DColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DOUBLE_ARRAY, true).addDoubleArray( + new double[][]{{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}} + ); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDoubleArray3DColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DOUBLE_ARRAY, true).addDoubleArray( + new double[][][]{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} + ); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDoubleArrayColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DOUBLE_ARRAY, true).addDoubleArray(new double[]{1.0, 2.0, 3.0}); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testDoubleColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_DOUBLE, false).addDouble(3.14); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testFloatColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_FLOAT, false).addFloat(3.14f); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testGeoHashColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_GEOHASH, true).addGeoHash(0x1234L, 20); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testIntColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_INT, false).addInt(42); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testLong256Column() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_LONG256, false).addLong256(1L, 2L, 3L, 4L); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testLongArray2DColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_LONG_ARRAY, true).addLongArray( + new long[][]{{10L, 20L, 30L}, {40L, 50L, 60L}} + ); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testLongArrayColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_LONG_ARRAY, true).addLongArray(new long[]{10L, 20L, 30L}); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testLongColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_LONG, false).addLong(42L); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testMultiByteUtf8() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("\u6e2c\u5b9a")) { // 測定 (3 bytes per char) + buf.getOrCreateColumn("\u6e29\u5ea6", TYPE_DOUBLE, false).addDouble(22.5); // 温度 + buf.getOrCreateColumn("\u30e1\u30e2", TYPE_STRING, true).addString("\u3053\u3093\u306b\u3061\u306f"); // メモ, こんにちは + buf.getOrCreateColumn("\u5730\u57df", TYPE_SYMBOL, true).addSymbol("\u6771\u4eac"); // 地域, 東京 + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testMultiRowDoubleTimestamp() throws Exception { + assertMemoryLeak(() -> { + for (int rowCount : new int[]{1, 5, 10, 50}) { + try (QwpTableBuffer buf = new QwpTableBuffer("measurements")) { + for (int i = 0; i < rowCount; i++) { + buf.getOrCreateColumn("value", TYPE_DOUBLE, false).addDouble(i * 1.1); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); + buf.nextRow(); + } + assertEstimateAccuracy(buf, rowCount); + } + } + }); + } + + @Test + public void testNullableColumnMixedNullNonNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer idCol = buf.getOrCreateColumn("id", TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer valCol = buf.getOrCreateColumn("val", TYPE_DOUBLE, true); + QwpTableBuffer.ColumnBuffer tsCol = buf.getOrCreateColumn("", TYPE_TIMESTAMP, true); + + // Row 1: has value + idCol.addLong(1L); + valCol.addDouble(10.0); + tsCol.addLong(1_000_000L); + buf.nextRow(); + + // Row 2: null (skip val) + idCol.addLong(2L); + tsCol.addLong(2_000_000L); + buf.nextRow(); + + // Row 3: has value + idCol.addLong(3L); + valCol.addDouble(30.0); + tsCol.addLong(3_000_000L); + buf.nextRow(); + + // Row 4: null + idCol.addLong(4L); + tsCol.addLong(4_000_000L); + buf.nextRow(); + + // Row 5: has value + idCol.addLong(5L); + valCol.addDouble(50.0); + tsCol.addLong(5_000_000L); + buf.nextRow(); + + assertEstimateAccuracy(buf, 5); + } + }); + } + + @Test + public void testShortColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_SHORT, false).addShort((short) 42); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testStringColumnEmpty() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_STRING, true).addString(""); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testStringColumnLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_STRING, true).addString("a]".repeat(500)); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testStringColumnShort() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_STRING, true).addString("hello"); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testSymbolWith100DistinctValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + for (int i = 0; i < 100; i++) { + buf.getOrCreateColumn("sym", TYPE_SYMBOL, true).addSymbol("val-" + i); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); + buf.nextRow(); + } + assertEstimateAccuracy(buf, 100); + } + }); + } + + @Test + public void testSymbolWith10DistinctValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + for (int i = 0; i < 10; i++) { + buf.getOrCreateColumn("sym", TYPE_SYMBOL, true).addSymbol("value-" + i); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); + buf.nextRow(); + } + assertEstimateAccuracy(buf, 10); + } + }); + } + + @Test + public void testSymbolWith1DistinctValue() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + for (int i = 0; i < 5; i++) { + buf.getOrCreateColumn("sym", TYPE_SYMBOL, true).addSymbol("only-one"); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); + buf.nextRow(); + } + assertEstimateAccuracy(buf, 5); + } + }); + } + + @Test + public void testTimestampColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_TIMESTAMP, true).addLong(1_700_000_000L); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testTimestampNanosColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_TIMESTAMP_NANOS, true).addLong(1_700_000_000_000L); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testUuidColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_UUID, false).addUuid(123L, 456L); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testRandomSchemas() throws Exception { + assertMemoryLeak(() -> { + Random rng = new Random(42); + byte[] fixedTypes = { + TYPE_BOOLEAN, TYPE_BYTE, TYPE_SHORT, TYPE_INT, TYPE_LONG, + TYPE_FLOAT, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, + TYPE_UUID, TYPE_LONG256 + }; + + for (int trial = 0; trial < 100; trial++) { + int numCols = 1 + rng.nextInt(5); + int numRows = 1 + rng.nextInt(20); + + try (QwpTableBuffer buf = new QwpTableBuffer("random_" + trial)) { + byte[] colTypes = new byte[numCols]; + for (int c = 0; c < numCols; c++) { + int pick = rng.nextInt(fixedTypes.length + 2); + if (pick < fixedTypes.length) { + colTypes[c] = fixedTypes[pick]; + } else if (pick == fixedTypes.length) { + colTypes[c] = TYPE_STRING; + } else { + colTypes[c] = TYPE_SYMBOL; + } + } + + for (int row = 0; row < numRows; row++) { + for (int c = 0; c < numCols; c++) { + boolean isNullable = colTypes[c] == TYPE_STRING || colTypes[c] == TYPE_SYMBOL + || colTypes[c] == TYPE_TIMESTAMP; + QwpTableBuffer.ColumnBuffer col = buf.getOrCreateColumn( + "c" + c, colTypes[c], isNullable + ); + addRandomValue(col, colTypes[c], rng); + } + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + row); + buf.nextRow(); + } + assertEstimateAccuracy(buf, numRows); + } + } + }); + } + + @Test + public void testVarcharColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buf = new QwpTableBuffer("t")) { + buf.getOrCreateColumn("v", TYPE_VARCHAR, true).addString("varchar value"); + buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); + buf.nextRow(); + assertEstimateAccuracy(buf, 1); + } + }); + } + + @Test + public void testVarintSize() { + Assert.assertEquals(1, QwpDatagramSizeEstimator.varintSize(0)); + Assert.assertEquals(1, QwpDatagramSizeEstimator.varintSize(1)); + Assert.assertEquals(1, QwpDatagramSizeEstimator.varintSize(127)); + Assert.assertEquals(2, QwpDatagramSizeEstimator.varintSize(128)); + Assert.assertEquals(2, QwpDatagramSizeEstimator.varintSize(16383)); + Assert.assertEquals(3, QwpDatagramSizeEstimator.varintSize(16384)); + } + + private static void addRandomValue(QwpTableBuffer.ColumnBuffer col, byte type, Random rng) { + switch (type) { + case TYPE_BOOLEAN -> col.addBoolean(rng.nextBoolean()); + case TYPE_BYTE -> col.addByte((byte) rng.nextInt()); + case TYPE_SHORT -> col.addShort((short) rng.nextInt()); + case TYPE_INT -> col.addInt(rng.nextInt()); + case TYPE_LONG -> col.addLong(rng.nextLong()); + case TYPE_FLOAT -> col.addFloat(rng.nextFloat()); + case TYPE_DOUBLE -> col.addDouble(rng.nextDouble()); + case TYPE_DATE -> col.addLong(rng.nextLong()); + case TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> col.addLong(Math.abs(rng.nextLong())); + case TYPE_UUID -> col.addUuid(rng.nextLong(), rng.nextLong()); + case TYPE_LONG256 -> col.addLong256(rng.nextLong(), rng.nextLong(), rng.nextLong(), rng.nextLong()); + case TYPE_STRING -> col.addString("str" + rng.nextInt(1000)); + case TYPE_SYMBOL -> col.addSymbol("sym" + rng.nextInt(20)); + } + } + + private static void assertEstimateAccuracy(QwpTableBuffer buf, int rowCount) { + long estimate = QwpDatagramSizeEstimator.estimate(buf, rowCount); + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(false); + int actual = encoder.encode(buf, false); + + Assert.assertTrue( + "estimate (" + estimate + ") < actual (" + actual + ")", + estimate >= actual + ); + Assert.assertTrue( + "estimate (" + estimate + ") - actual (" + actual + ") = " + (estimate - actual) + " >= 32", + estimate - actual < 32 + ); + } + } +} From 1002c26b9556f4272c817f0cc6d52d493169a97a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 4 Mar 2026 16:33:50 +0100 Subject: [PATCH 142/230] Magic bytes: ILP4 -> QWP1 --- .../client/cutlass/qwp/client/QwpWebSocketEncoder.java | 6 +++--- .../questdb/client/cutlass/qwp/protocol/QwpConstants.java | 4 ++-- .../test/cutlass/qwp/client/NativeBufferWriterTest.java | 8 ++++---- .../test/cutlass/qwp/protocol/QwpConstantsTest.java | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 1177add..632e4c8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -126,10 +126,10 @@ public void setGorillaEnabled(boolean enabled) { } public void writeHeader(int tableCount, int payloadLength) { - buffer.putByte((byte) 'I'); - buffer.putByte((byte) 'L'); + buffer.putByte((byte) 'Q'); + buffer.putByte((byte) 'W'); buffer.putByte((byte) 'P'); - buffer.putByte((byte) '4'); + buffer.putByte((byte) '1'); buffer.putByte(VERSION_1); buffer.putByte(flags); buffer.putShort((short) tableCount); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 8c36d93..216f2fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -94,9 +94,9 @@ public final class QwpConstants { */ public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian /** - * Magic bytes for ILP v4 message: "ILP4" (ASCII). + * Magic bytes for QWP v1 message: "QWP1" (ASCII). */ - public static final int MAGIC_MESSAGE = 0x34504C49; // "ILP4" in little-endian + public static final int MAGIC_MESSAGE = 0x31505751; // "QWP1" in little-endian /** * Maximum columns per table (QuestDB limit). */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index f3ea891..a2cda57 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -77,11 +77,11 @@ public void testMultipleWrites() throws Exception { Assert.assertEquals(12, writer.getPosition()); - // Verify ILP4 header - Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); - Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + // Verify QWP1 header + Assert.assertEquals((byte) 'Q', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'W', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); - Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + Assert.assertEquals((byte) '1', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); } }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java index 2b6ce04..f8654d5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -163,12 +163,12 @@ public void testMagicBytesFallback() { @Test public void testMagicBytesValue() { - // "ILP4" in ASCII: I=0x49, L=0x4C, P=0x50, 4=0x34 - // Little-endian: 0x34504C49 - Assert.assertEquals(0x34504C49, MAGIC_MESSAGE); + // "QWP1" in ASCII: Q=0x51, W=0x57, P=0x50, 1=0x31 + // Little-endian: 0x31505751 + Assert.assertEquals(0x31505751, MAGIC_MESSAGE); // Verify ASCII encoding - byte[] expected = new byte[]{'I', 'L', 'P', '4'}; + byte[] expected = new byte[]{'Q', 'W', 'P', '1'}; Assert.assertEquals((byte) (MAGIC_MESSAGE & 0xFF), expected[0]); Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 8) & 0xFF), expected[1]); Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 16) & 0xFF), expected[2]); From 6f566ad986f337a394f2c9dec10d94b5a0cbe7fb Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 4 Mar 2026 16:49:14 +0100 Subject: [PATCH 143/230] iteration #4 --- .../main/java/io/questdb/client/Sender.java | 193 ++++++++- .../cutlass/line/LineSenderBuilderTest.java | 12 +- .../qwp/client/LineSenderBuilderUdpTest.java | 377 ++++++++++++++++++ .../LineSenderBuilderWebSocketTest.java | 2 +- 4 files changed, 573 insertions(+), 11 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index c815bf7..4ec6a65 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -34,6 +34,7 @@ import io.questdb.client.cutlass.line.http.AbstractLineHttpSender; import io.questdb.client.cutlass.line.tcp.DelegatingTlsChannel; import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; +import io.questdb.client.cutlass.qwp.client.QwpUdpSender; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.network.NetworkFacade; @@ -52,6 +53,8 @@ import javax.security.auth.DestroyFailedException; import java.io.Closeable; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.security.PrivateKey; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -148,6 +151,7 @@ static LineSenderBuilder builder(Transport transport) { int protocol = switch (transport) { case HTTP -> LineSenderBuilder.PROTOCOL_HTTP; case TCP -> LineSenderBuilder.PROTOCOL_TCP; + case UDP -> LineSenderBuilder.PROTOCOL_UDP; case WEBSOCKET -> LineSenderBuilder.PROTOCOL_WEBSOCKET; }; return new LineSenderBuilder(protocol); @@ -469,6 +473,14 @@ enum Transport { */ TCP, + /** + * Fire-and-forget binary ingestion over UDP. + *

    + * UDP transport sends datagrams without waiting for acknowledgement. It is suitable for + * high-throughput scenarios where occasional message loss is acceptable. + */ + UDP, + /** * Use WebSocket transport to communicate with a QuestDB server. *

    @@ -520,6 +532,7 @@ final class LineSenderBuilder { private static final int DEFAULT_AUTO_FLUSH_INTERVAL_MILLIS = 1_000; private static final int DEFAULT_AUTO_FLUSH_ROWS = 75_000; private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024; + private static final int DEFAULT_MAX_DATAGRAM_SIZE = 1400; private static final int DEFAULT_HTTP_PORT = 9000; private static final int DEFAULT_HTTP_TIMEOUT = 30_000; private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8; @@ -529,6 +542,7 @@ final class LineSenderBuilder { private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method private static final int DEFAULT_TCP_PORT = 9009; + private static final int DEFAULT_UDP_PORT = 9007; private static final int DEFAULT_WEBSOCKET_PORT = 9000; private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms @@ -541,6 +555,7 @@ final class LineSenderBuilder { private static final int PARAMETER_NOT_SET_EXPLICITLY = -1; private static final int PROTOCOL_HTTP = 1; private static final int PROTOCOL_TCP = 0; + private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); @@ -556,8 +571,10 @@ final class LineSenderBuilder { private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY; private String keyId; private int maxBackoffMillis = PARAMETER_NOT_SET_EXPLICITLY; + private int maxDatagramSize = PARAMETER_NOT_SET_EXPLICITLY; private int maxNameLength = PARAMETER_NOT_SET_EXPLICITLY; private int maximumBufferCapacity = PARAMETER_NOT_SET_EXPLICITLY; + private int multicastTtl = PARAMETER_NOT_SET_EXPLICITLY; private final HttpClientConfiguration httpClientConfiguration = new DefaultHttpClientConfiguration() { @Override public int getInitialRequestBufferSize() { @@ -889,6 +906,17 @@ public Sender build() { } } + if (protocol == PROTOCOL_UDP) { + if (hosts.size() != 1 || ports.size() != 1) { + throw new LineSenderException("only a single address (host:port) is supported for UDP transport"); + } + int sendToAddr = resolveIPv4(hosts.getQuick(0)); + int actualMaxDatagramSize = maxDatagramSize == PARAMETER_NOT_SET_EXPLICITLY + ? DEFAULT_MAX_DATAGRAM_SIZE : maxDatagramSize; + int actualTtl = multicastTtl == PARAMETER_NOT_SET_EXPLICITLY ? 0 : multicastTtl; + return new QwpUdpSender(nf, 0, sendToAddr, ports.getQuick(0), actualTtl, actualMaxDatagramSize); + } + assert protocol == PROTOCOL_TCP; if (hosts.size() != 1 || ports.size() != 1) { @@ -1209,6 +1237,32 @@ public LineSenderBuilder maxBackoffMillis(int maxBackoffMillis) { return this; } + /** + * Set the maximum datagram size in bytes for UDP transport. Only valid for UDP transport. + *
    + * The practical limit depends on the network MTU (typically 1500 bytes for Ethernet). + *
    + * Default value: 1400 bytes + * + * @param maxDatagramSize maximum datagram size in bytes + * @return this instance for method chaining + */ + public LineSenderBuilder maxDatagramSize(int maxDatagramSize) { + if (this.maxDatagramSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("max datagram size was already configured ") + .put("[maxDatagramSize=").put(this.maxDatagramSize).put("]"); + } + if (maxDatagramSize < 1) { + throw new LineSenderException("max datagram size must be positive ") + .put("[maxDatagramSize=").put(maxDatagramSize).put("]"); + } + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_UDP) { + throw new LineSenderException("max datagram size is only supported for UDP transport"); + } + this.maxDatagramSize = maxDatagramSize; + return this; + } + /** * Set the maximum local buffer capacity in bytes. *
    @@ -1282,6 +1336,36 @@ public LineSenderBuilder minRequestThroughput(int minRequestThroughput) { return this; } + /** + * Set the multicast TTL for UDP transport. Only valid for UDP transport. + *
    + * Valid range: 0-255. + *
    + * Default value: 0 (restricted to same host). Set to 1 for local subnet. + * + * @param multicastTtl multicast TTL value + * @return this instance for method chaining + */ + public LineSenderBuilder multicastTtl(int multicastTtl) { + if (this.multicastTtl != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("multicast TTL was already configured ") + .put("[multicastTtl=").put(this.multicastTtl).put("]"); + } + if (multicastTtl < 0) { + throw new LineSenderException("multicast TTL cannot be negative ") + .put("[multicastTtl=").put(multicastTtl).put("]"); + } + if (multicastTtl > 255) { + throw new LineSenderException("multicast TTL cannot exceed 255 ") + .put("[multicastTtl=").put(multicastTtl).put("]"); + } + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_UDP) { + throw new LineSenderException("multicast TTL is only supported for UDP transport"); + } + this.multicastTtl = multicastTtl; + return this; + } + /** * Set port where a QuestDB server is listening on. * @@ -1378,6 +1462,21 @@ private static int parseIntValue(@NotNull StringSink value, @NotNull String name } } + private static int resolveIPv4(String host) { + try { + byte[] addr = InetAddress.getByName(host).getAddress(); + if (addr.length != 4) { + throw new LineSenderException("IPv6 addresses are not supported [host=").put(host).put("]"); + } + return ((addr[0] & 0xFF) << 24) + | ((addr[1] & 0xFF) << 16) + | ((addr[2] & 0xFF) << 8) + | (addr[3] & 0xFF); + } catch (UnknownHostException e) { + throw new LineSenderException("could not resolve host [host=" + host + "]", e); + } + } + private static RuntimeException rethrow(Throwable t) { if (t instanceof LineSenderException) { throw (LineSenderException) t; @@ -1398,6 +1497,8 @@ private void configureDefaults() { if (ports.size() == 0) { if (protocol == PROTOCOL_HTTP) { ports.add(DEFAULT_HTTP_PORT); + } else if (protocol == PROTOCOL_UDP) { + ports.add(DEFAULT_UDP_PORT); } else if (protocol == PROTOCOL_WEBSOCKET) { ports.add(DEFAULT_WEBSOCKET_PORT); } else { @@ -1443,7 +1544,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol was already configured ") .put("[protocol=") - .put(protocol == PROTOCOL_HTTP ? "http" : "tcp").put("]"); + .put(switch (protocol) { + case PROTOCOL_HTTP -> "http"; + case PROTOCOL_UDP -> "udp"; + case PROTOCOL_WEBSOCKET -> "websocket"; + default -> "tcp"; + }).put("]"); } if (Chars.equals("http", sink)) { if (tlsEnabled) { @@ -1469,8 +1575,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else if (Chars.equals("wss", sink)) { websocket(); tlsEnabled = true; + } else if (Chars.equals("udp", sink)) { + udp(); + } else if (Chars.equals("udps", sink)) { + throw new LineSenderException("TLS is not supported for UDP"); } else { - throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); } String tcpToken = null; @@ -1492,28 +1602,39 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { address(sink); if (ports.size() == hosts.size() - 1) { // not set - port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT + port(protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT + : protocol == PROTOCOL_UDP ? DEFAULT_UDP_PORT : protocol == PROTOCOL_WEBSOCKET ? DEFAULT_WEBSOCKET_PORT - : DEFAULT_HTTP_PORT); + : DEFAULT_TCP_PORT); } } else if (Chars.equals("user", sink)) { // deprecated key: user, new key: username pos = getValue(configurationString, pos, sink, "user"); + if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("username is not supported for UDP transport"); + } user = sink.toString(); } else if (Chars.equals("username", sink)) { pos = getValue(configurationString, pos, sink, "username"); + if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("username is not supported for UDP transport"); + } user = sink.toString(); } else if (Chars.equals("pass", sink)) { // deprecated key: pass, new key: password pos = getValue(configurationString, pos, sink, "pass"); if (protocol == PROTOCOL_TCP) { throw new LineSenderException("password is not supported for TCP protocol"); + } else if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("password is not supported for UDP transport"); } password = sink.toString(); } else if (Chars.equals("password", sink)) { pos = getValue(configurationString, pos, sink, "password"); if (protocol == PROTOCOL_TCP) { throw new LineSenderException("password is not supported for TCP protocol"); + } else if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("password is not supported for UDP transport"); } password = sink.toString(); } else if (Chars.equals("tls_verify", sink)) { @@ -1550,6 +1671,8 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { // will configure later, we need to know a keyId first } else if (protocol == PROTOCOL_HTTP) { httpToken(sink.toString()); + } else if (protocol == PROTOCOL_UDP) { + throw new LineSenderException("token is not supported for UDP transport"); } else { throw new LineSenderException("token is not supported for WebSocket protocol"); } @@ -1640,6 +1763,14 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { int protocolVersion = parseIntValue(sink, "protocol_version"); protocolVersion(protocolVersion); } + } else if (Chars.equals("max_datagram_size", sink)) { + pos = getValue(configurationString, pos, sink, "max_datagram_size"); + int mds = parseIntValue(sink, "max_datagram_size"); + maxDatagramSize(mds); + } else if (Chars.equals("multicast_ttl", sink)) { + pos = getValue(configurationString, pos, sink, "multicast_ttl"); + int ttl = parseIntValue(sink, "multicast_ttl"); + multicastTtl(ttl); } else { // ignore unknown keys, unless they are malformed if ((pos = ConfStringParser.value(configurationString, pos, sink)) < 0) { @@ -1754,6 +1885,52 @@ private void validateParameters() { if (autoFlushIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("auto flush interval is not supported for TCP protocol"); } + } else if (protocol == PROTOCOL_UDP) { + if (privateKey != null) { + throw new LineSenderException("authentication is not supported for UDP transport"); + } + if (httpToken != null) { + throw new LineSenderException("HTTP token authentication is not supported for UDP transport"); + } + if (username != null || password != null) { + throw new LineSenderException("username/password authentication is not supported for UDP transport"); + } + if (tlsEnabled) { + throw new LineSenderException("TLS is not supported for UDP transport"); + } + if (retryTimeoutMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("retry timeout is not supported for UDP transport"); + } + if (httpTimeout != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("HTTP timeout is not supported for UDP transport"); + } + if (minRequestThroughput != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("minimum request throughput is not supported for UDP transport"); + } + if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol version is not supported for UDP transport"); + } + if (asyncMode) { + throw new LineSenderException("async mode is not supported for UDP transport"); + } + if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("in-flight window size is not supported for UDP transport"); + } + if (httpPath != null) { + throw new LineSenderException("HTTP path is not supported for UDP transport"); + } + if (maxBackoffMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("max backoff is not supported for UDP transport"); + } + if (autoFlushRows != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush rows is not supported for UDP transport"); + } + if (autoFlushIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush interval is not supported for UDP transport"); + } + if (autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush bytes is not supported for UDP transport"); + } } else if (protocol == PROTOCOL_WEBSOCKET) { if (privateKey != null) { throw new LineSenderException("TCP authentication is not supported for WebSocket protocol"); @@ -1792,6 +1969,14 @@ private void validateParameters() { } } + private void udp() { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol was already configured ") + .put("[protocol=").put(protocol).put("]"); + } + protocol = PROTOCOL_UDP; + } + private void websocket() { if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol was already configured ") diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java index 031f204..a47cb34 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -140,8 +140,8 @@ public void testBufferSizeDoubleSet() throws Exception { @Test public void testConfStringValidation() throws Exception { assertMemoryLeak(() -> { - assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); - assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); assertConfStrError("http::addr=localhost:-1;", "invalid port [port=-1]"); assertConfStrError("http::auto_flush=on;", "addr is missing"); assertConfStrError("http::addr=localhost;tls_roots=/some/path;", "tls_roots was configured, but tls_roots_password is missing"); @@ -171,10 +171,10 @@ public void testConfStringValidation() throws Exception { assertConfStrError("http::addr=localhost:8080;auto_flush=invalid;", "invalid auto_flush [value=invalid, allowed-values=[on, off]]"); assertConfStrError("http::addr=localhost:8080;auto_flush=off;auto_flush_rows=100;", "cannot set auto flush rows when auto-flush is already disabled"); assertConfStrError("http::addr=localhost:8080;auto_flush_rows=100;auto_flush=off;", "auto flush rows was already configured [autoFlushRows=100]"); - assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); - assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); - assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); - assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); + assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_interval=1;", "cannot set auto flush interval when interval based auto-flush is already disabled"); assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_rows=1;", "cannot set auto flush rows when auto-flush is already disabled"); assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java new file mode 100644 index 0000000..632a549 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java @@ -0,0 +1,377 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpUdpSender; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for UDP transport support in the Sender.builder() API. + * These tests verify the builder configuration and validation, + * not actual UDP connectivity (which requires a running server). + */ +public class LineSenderBuilderUdpTest extends AbstractTest { + + @Test + public void testInvalidSchema_includesUdp() { + assertBadConfig("invalid::addr=localhost:9000;", "udp"); + } + + @Test + public void testUdpScheme_buildsQwpUdpSender() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost:9007;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdpScheme_customMaxDatagramSize() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost:9007;max_datagram_size=500;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdpScheme_customMulticastTtl() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost:9007;multicast_ttl=5;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdpScheme_noAddr_throws() { + assertBadConfig("udp::foo=bar;", "addr is missing"); + } + + @Test + public void testUdp_authNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .enableAuth("keyId") + .authToken("5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), + "not supported for UDP"); + } + + @Test + public void testUdp_asyncModeNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .asyncMode(true), + "not supported for UDP"); + } + + @Test + public void testUdp_autoFlushBytesNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .autoFlushBytes(1000), + "not supported for UDP"); + } + + @Test + public void testUdp_autoFlushIntervalNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .autoFlushIntervalMillis(100), + "not supported for UDP"); + } + + @Test + public void testUdp_autoFlushRowsNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .autoFlushRows(100), + "not supported for UDP"); + } + + @Test + public void testUdp_defaultPort9007() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.fromConfig("udp::addr=localhost;")) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdp_httpPathNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpPath("/custom/path"), + "not supported for UDP"); + } + + @Test + public void testUdp_httpTimeoutNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpTimeoutMillis(5000), + "not supported for UDP"); + } + + @Test + public void testUdp_httpTokenNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpToken("token"), + "not supported for UDP"); + } + + @Test + public void testUdp_inFlightWindowSizeNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .inFlightWindowSize(1000), + "not supported for UDP"); + } + + @Test + public void testUdp_maxBackoffNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxBackoffMillis(5000), + "not supported for UDP"); + } + + @Test + public void testUdp_maxDatagramSize() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxDatagramSize(500); + Assert.assertNotNull(builder); + } + + @Test + public void testUdp_maxDatagramSizeDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxDatagramSize(500) + .maxDatagramSize(600)); + } + + @Test + public void testUdp_maxDatagramSizeZero_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .maxDatagramSize(0)); + } + + @Test + public void testUdp_minRequestThroughputNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .minRequestThroughput(100), + "not supported for UDP"); + } + + @Test + public void testUdp_multicastTtl() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(5); + Assert.assertNotNull(builder); + } + + @Test + public void testUdp_multicastTtlDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(5) + .multicastTtl(10)); + } + + @Test + public void testUdp_multicastTtlNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(-1)); + } + + @Test + public void testUdp_protocolVersionNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .protocolVersion(1), + "not supported for UDP"); + } + + @Test + public void testUdp_ipv6Address_throws() { + assertThrowsAny( + () -> Sender.builder(Sender.Transport.UDP) + .address("::1"), + "cannot parse a port", "use IPv4 address"); + } + + @Test + public void testUdp_resolveUnknownHost_throws() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("this.host.does.not.exist.questdb.invalid"), + "could not resolve host"); + } + + @Test + public void testUdp_retryTimeoutNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .retryTimeoutMillis(100), + "not supported for UDP"); + } + + @Test + public void testUdp_tlsEnabled_throws() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .enableTls(), + "TLS is not supported for UDP"); + } + + @Test + public void testUdp_tokenNotSupported() { + assertBadConfig("udp::addr=localhost:9007;token=foo;", "token is not supported for UDP"); + } + + @Test + public void testUdp_transportEnum() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender sender = Sender.builder(Sender.Transport.UDP) + .address("localhost:9007") + .build()) { + Assert.assertTrue(sender instanceof QwpUdpSender); + } + }); + } + + @Test + public void testUdp_usernamePasswordNotSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.UDP) + .address("localhost") + .httpUsernamePassword("user", "pass"), + "not supported for UDP"); + } + + @Test + public void testUdpScheme_password_throws() { + assertBadConfig("udp::addr=localhost;password=bar;", "password is not supported for UDP"); + } + + @Test + public void testUdpScheme_username_throws() { + assertBadConfig("udp::addr=localhost;username=foo;", "username is not supported for UDP"); + } + + @Test + public void testUdp_maxDatagramSizeNonUdp_fails() { + assertThrows("only supported for UDP transport", + () -> Sender.builder(Sender.Transport.HTTP) + .address("localhost") + .maxDatagramSize(500)); + } + + @Test + public void testUdp_multicastTtlExceeds255_fails() { + assertThrows("cannot exceed 255", + () -> Sender.builder(Sender.Transport.UDP) + .address("localhost") + .multicastTtl(256)); + } + + @Test + public void testUdp_multicastTtlNonUdp_fails() { + assertThrows("only supported for UDP transport", + () -> Sender.builder(Sender.Transport.HTTP) + .address("localhost") + .multicastTtl(1)); + } + + @Test + public void testUdps_throws() { + assertBadConfig("udps::addr=localhost:9007;", "TLS is not supported for UDP"); + } + + @SuppressWarnings("resource") + private static void assertBadConfig(String config, String... anyOf) { + assertThrowsAny(() -> Sender.fromConfig(config), anyOf); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + Assert.fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void assertThrowsAny(Sender.LineSenderBuilder builder, String... anyOf) { + assertThrowsAny(builder::build, anyOf); + } + + private static void assertThrowsAny(Runnable action, String... anyOf) { + try { + action.run(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + for (String s : anyOf) { + if (msg.contains(s)) { + return; + } + } + Assert.fail("Expected message containing one of [" + String.join(", ", anyOf) + "] but got: " + msg); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index ae6480c..ec05aa4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -433,7 +433,7 @@ public void testInvalidPort_fails() { @Test public void testInvalidSchema_fails() { - assertBadConfig("invalid::addr=localhost:9000;", "invalid schema [schema=invalid, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertBadConfig("invalid::addr=localhost:9000;", "invalid schema [schema=invalid, supported-schemas=[http, https, tcp, tcps, ws, wss, udp]]"); } @Test From 6e4d10074d47d949d2387fa63bcfcea6eefa2b79 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 4 Mar 2026 17:50:51 +0100 Subject: [PATCH 144/230] iteration #5 --- .../cutlass/qwp/client/QwpColumnWriter.java | 343 +++++++++++++++++ .../cutlass/qwp/client/QwpUdpSender.java | 34 +- .../qwp/client/QwpWebSocketEncoder.java | 364 +----------------- 3 files changed, 376 insertions(+), 365 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java new file mode 100644 index 0000000..d1ab0a4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -0,0 +1,343 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Transport-agnostic column encoder for ILP v4 table data. + *

    + * Reads column data from {@link QwpTableBuffer.ColumnBuffer} and writes encoded + * bytes to a {@link QwpBufferWriter}. Both {@link QwpWebSocketEncoder} and + * {@link QwpUdpSender} delegate to this class for column encoding. + */ +class QwpColumnWriter { + + private static final byte ENCODING_GORILLA = 0x01; + private static final byte ENCODING_UNCOMPRESSED = 0x00; + private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); + private QwpBufferWriter buffer; + + private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla, boolean useGlobalSymbols) { + int valueCount = col.getValueCount(); + long dataAddr = col.getDataAddress(); + + if (colDef.isNullable()) { + writeNullBitmap(col, rowCount); + } + + switch (col.getType()) { + case TYPE_BOOLEAN: + writeBooleanColumn(dataAddr, valueCount); + break; + case TYPE_BYTE: + buffer.putBlockOfBytes(dataAddr, valueCount); + break; + case TYPE_SHORT: + case TYPE_CHAR: + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); + break; + case TYPE_INT, TYPE_FLOAT: + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); + break; + case TYPE_LONG, TYPE_DATE, TYPE_DOUBLE: + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); + break; + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + writeTimestampColumn(dataAddr, valueCount, useGorilla); + break; + case TYPE_GEOHASH: + writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + writeStringColumn(col, valueCount); + break; + case TYPE_SYMBOL: + if (useGlobalSymbols) { + writeSymbolColumnWithGlobalIds(col, valueCount); + } else { + writeSymbolColumn(col, valueCount); + } + break; + case TYPE_UUID: + // Stored as lo+hi contiguously, matching wire order + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); + break; + case TYPE_LONG256: + // Stored as 4 contiguous longs per value + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); + break; + case TYPE_DOUBLE_ARRAY: + writeDoubleArrayColumn(col, valueCount); + break; + case TYPE_LONG_ARRAY: + writeLongArrayColumn(col, valueCount); + break; + case TYPE_DECIMAL64: + writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); + break; + case TYPE_DECIMAL128: + writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); + break; + case TYPE_DECIMAL256: + writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); + break; + default: + throw new LineSenderException("Unknown column type: " + col.getType()); + } + } + + void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols, boolean useGorilla) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + QwpColumnDef colDef = columnDefs[i]; + encodeColumn(col, colDef, rowCount, useGorilla, useGlobalSymbols); + } + } + + void setBuffer(QwpBufferWriter buffer) { + this.buffer = buffer; + } + + private void writeBooleanColumn(long addr, int count) { + int packedSize = (count + 7) / 8; + for (int i = 0; i < packedSize; i++) { + byte b = 0; + for (int bit = 0; bit < 8; bit++) { + int idx = i * 8 + bit; + if (idx < count && Unsafe.getUnsafe().getByte(addr + idx) != 0) { + b |= (1 << bit); + } + } + buffer.putByte(b); + } + } + + private void writeDecimal128Column(byte scale, long addr, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + long offset = (long) i * 16; + long hi = Unsafe.getUnsafe().getLong(addr + offset); + long lo = Unsafe.getUnsafe().getLong(addr + offset + 8); + buffer.putLongBE(hi); + buffer.putLongBE(lo); + } + } + + private void writeDecimal256Column(byte scale, long addr, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + long offset = (long) i * 32; + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 8)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 16)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 24)); + } + } + + private void writeDecimal64Column(byte scale, long addr, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + (long) i * 8)); + } + } + + private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount = Math.multiplyExact(elemCount, dimLen); + } + + for (int e = 0; e < elemCount; e++) { + buffer.putDouble(data[dataIdx++]); + } + } + } + + private void writeGeoHashColumn(long addr, int count, int precision) { + if (precision < 1) { + precision = 1; + } + buffer.putVarint(precision); + int valueSize = (precision + 7) / 8; + for (int i = 0; i < count; i++) { + long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8); + for (int b = 0; b < valueSize; b++) { + buffer.putByte((byte) (value >>> (b * 8))); + } + } + } + + private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount = Math.multiplyExact(elemCount, dimLen); + } + + for (int e = 0; e < elemCount; e++) { + buffer.putLong(data[dataIdx++]); + } + } + } + + private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) { + long nullAddr = col.getNullBitmapAddress(); + if (nullAddr != 0) { + int bitmapSize = (rowCount + 7) / 8; + buffer.putBlockOfBytes(nullAddr, bitmapSize); + } else { + int bitmapSize = (rowCount + 7) / 8; + for (int i = 0; i < bitmapSize; i++) { + buffer.putByte((byte) 0); + } + } + } + + private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) { + buffer.putBlockOfBytes(col.getStringOffsetsAddress(), (long) (valueCount + 1) * 4); + buffer.putBlockOfBytes(col.getStringDataAddress(), col.getStringDataSize()); + } + + private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) { + long dataAddr = col.getDataAddress(); + String[] dictionary = col.getSymbolDictionary(); + + buffer.putVarint(dictionary.length); + for (String symbol : dictionary) { + buffer.putString(symbol); + } + + for (int i = 0; i < count; i++) { + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); + } + } + + private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) { + long auxAddr = col.getAuxDataAddress(); + if (auxAddr == 0) { + long dataAddr = col.getDataAddress(); + for (int i = 0; i < count; i++) { + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); + } + } else { + for (int i = 0; i < count; i++) { + int globalId = Unsafe.getUnsafe().getInt(auxAddr + (long) i * 4); + buffer.putVarint(globalId); + } + } + } + + private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) { + buffer.putString(tableName); + buffer.putVarint(rowCount); + buffer.putVarint(columns.length); + buffer.putByte(SCHEMA_MODE_FULL); + for (QwpColumnDef col : columns) { + buffer.putString(col.getName()); + buffer.putByte(col.getWireTypeCode()); + } + } + + private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { + buffer.putString(tableName); + buffer.putVarint(rowCount); + buffer.putVarint(columnCount); + buffer.putByte(SCHEMA_MODE_REFERENCE); + buffer.putLong(schemaHash); + } + + private void writeTimestampColumn(long addr, int count, boolean useGorilla) { + if (useGorilla && count > 2) { + if (QwpGorillaEncoder.canUseGorilla(addr, count)) { + buffer.putByte(ENCODING_GORILLA); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(addr, count); + buffer.ensureCapacity(encodedSize); + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + addr, + count + ); + buffer.skip(bytesWritten); + } else { + buffer.putByte(ENCODING_UNCOMPRESSED); + buffer.putBlockOfBytes(addr, (long) count * 8); + } + } else { + if (useGorilla) { + buffer.putByte(ENCODING_UNCOMPRESSED); + } + buffer.putBlockOfBytes(addr, (long) count * 8); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 5255daa..a82afc3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -75,8 +75,9 @@ public class QwpUdpSender implements Sender { private static final byte ENTRY_TIMESTAMP_COL_NANOS = 14; private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); + private final NativeBufferWriter buffer = new NativeBufferWriter(); private final UdpLineChannel channel; - private final QwpWebSocketEncoder encoder; + private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private final int maxDatagramSize; private final ObjList rowJournal = new ObjList<>(); private final CharSequenceObjHashMap tableBuffers; @@ -93,8 +94,6 @@ public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int } public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl, int maxDatagramSize) { - this.encoder = new QwpWebSocketEncoder(); - this.encoder.setGorillaEnabled(false); this.channel = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); this.tableBuffers = new CharSequenceObjHashMap<>(); this.maxDatagramSize = maxDatagramSize; @@ -182,7 +181,7 @@ public void close() { } tableBuffers.clear(); channel.close(); - encoder.close(); + buffer.close(); } } @@ -571,6 +570,25 @@ private void checkTableSelected() { } } + private int encodeForUdp(QwpTableBuffer tableBuffer) { + buffer.reset(); + // Write 12-byte ILP4 header: magic, version, flags=0, tableCount=1, payloadLength=0 (patched later) + buffer.putByte((byte) 'I'); + buffer.putByte((byte) 'L'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '4'); + buffer.putByte(VERSION_1); + buffer.putByte((byte) 0); // flags + buffer.putShort((short) 1); // tableCount + buffer.putInt(0); // payloadLength placeholder + int payloadStart = buffer.getPosition(); + columnWriter.setBuffer(buffer); + columnWriter.encodeTable(tableBuffer, false, false, false); + int payloadLength = buffer.getPosition() - payloadStart; + buffer.patchInt(8, payloadLength); + return buffer.getPosition(); + } + private void flushInternal() { ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { @@ -579,9 +597,9 @@ private void flushInternal() { QwpTableBuffer tableBuffer = tableBuffers.get(tableName); if (tableBuffer == null || tableBuffer.getRowCount() == 0) continue; - int len = encoder.encode(tableBuffer, false); + int len = encodeForUdp(tableBuffer); try { - channel.send(encoder.getBuffer().getBufferPtr(), len); + channel.send(buffer.getBufferPtr(), len); } catch (LineSenderException e) { LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); } @@ -592,9 +610,9 @@ private void flushInternal() { } private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { - int len = encoder.encode(tableBuffer, false); + int len = encodeForUdp(tableBuffer); try { - channel.send(encoder.getBuffer().getBufferPtr(), len); + channel.send(buffer.getBufferPtr(), len); } catch (LineSenderException e) { LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 1177add..a7d1f9b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -24,33 +24,20 @@ package io.questdb.client.cutlass.qwp.client; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; -import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.Unsafe; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * Encodes ILP v4 messages for WebSocket transport. *

    - * This encoder reads column data from off-heap {@link io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory} - * buffers in {@link QwpTableBuffer.ColumnBuffer} and uses bulk {@code putBlockOfBytes} for fixed-width - * types where wire format matches native byte order. - *

    - * Types that use bulk copy (native byte-order on wire): - * BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DATE, UUID, LONG256 - *

    - * Types that require element-by-element encoding: - * BOOLEAN (bit-packed on wire), TIMESTAMP (Gorilla), DECIMAL64/128/256 (big-endian on wire) + * This encoder delegates column encoding to {@link QwpColumnWriter} and wraps + * the encoded payload with a 12-byte ILP4 header. */ public class QwpWebSocketEncoder implements QuietCloseable { - public static final byte ENCODING_GORILLA = 0x01; - public static final byte ENCODING_UNCOMPRESSED = 0x00; - private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); + private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; private byte flags; @@ -76,7 +63,8 @@ public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { buffer.reset(); writeHeader(1, 0); int payloadStart = buffer.getPosition(); - encodeTable(tableBuffer, useSchemaRef, false); + columnWriter.setBuffer(buffer); + columnWriter.encodeTable(tableBuffer, useSchemaRef, false, isGorillaEnabled()); int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); return buffer.getPosition(); @@ -102,7 +90,8 @@ public int encodeWithDeltaDict( String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } - encodeTable(tableBuffer, useSchemaRef, true); + columnWriter.setBuffer(buffer); + columnWriter.encodeTable(tableBuffer, useSchemaRef, true, isGorillaEnabled()); int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); flags = savedFlags; @@ -135,343 +124,4 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } - - private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla, boolean useGlobalSymbols) { - int valueCount = col.getValueCount(); - long dataAddr = col.getDataAddress(); - - if (colDef.isNullable()) { - writeNullBitmap(col, rowCount); - } - - switch (col.getType()) { - case TYPE_BOOLEAN: - writeBooleanColumn(dataAddr, valueCount); - break; - case TYPE_BYTE: - buffer.putBlockOfBytes(dataAddr, valueCount); - break; - case TYPE_SHORT: - case TYPE_CHAR: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); - break; - case TYPE_INT, TYPE_FLOAT: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); - break; - case TYPE_LONG, TYPE_DATE, TYPE_DOUBLE: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - writeTimestampColumn(dataAddr, valueCount, useGorilla); - break; - case TYPE_GEOHASH: - writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); - break; - case TYPE_STRING: - case TYPE_VARCHAR: - writeStringColumn(col, valueCount); - break; - case TYPE_SYMBOL: - if (useGlobalSymbols) { - writeSymbolColumnWithGlobalIds(col, valueCount); - } else { - writeSymbolColumn(col, valueCount); - } - break; - case TYPE_UUID: - // Stored as lo+hi contiguously, matching wire order - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); - break; - case TYPE_LONG256: - // Stored as 4 contiguous longs per value - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); - break; - case TYPE_DOUBLE_ARRAY: - writeDoubleArrayColumn(col, valueCount); - break; - case TYPE_LONG_ARRAY: - writeLongArrayColumn(col, valueCount); - break; - case TYPE_DECIMAL64: - writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); - break; - case TYPE_DECIMAL128: - writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); - break; - case TYPE_DECIMAL256: - writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); - break; - default: - throw new LineSenderException("Unknown column type: " + col.getType()); - } - } - - private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols) { - QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - int rowCount = tableBuffer.getRowCount(); - - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - tableBuffer.getSchemaHash(), - columnDefs.length - ); - } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); - } - - boolean useGorilla = isGorillaEnabled(); - for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - QwpColumnDef colDef = columnDefs[i]; - encodeColumn(col, colDef, rowCount, useGorilla, useGlobalSymbols); - } - } - - /** - * Writes boolean column data (bit-packed on wire). - * Reads individual bytes from off-heap and packs into bits. - */ - private void writeBooleanColumn(long addr, int count) { - int packedSize = (count + 7) / 8; - for (int i = 0; i < packedSize; i++) { - byte b = 0; - for (int bit = 0; bit < 8; bit++) { - int idx = i * 8 + bit; - if (idx < count && Unsafe.getUnsafe().getByte(addr + idx) != 0) { - b |= (1 << bit); - } - } - buffer.putByte(b); - } - } - - /** - * Writes Decimal128 values in big-endian wire format. - * Reads hi/lo pairs from off-heap (stored as hi, lo per value). - */ - private void writeDecimal128Column(byte scale, long addr, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - long offset = (long) i * 16; - long hi = Unsafe.getUnsafe().getLong(addr + offset); - long lo = Unsafe.getUnsafe().getLong(addr + offset + 8); - buffer.putLongBE(hi); - buffer.putLongBE(lo); - } - } - - /** - * Writes Decimal256 values in big-endian wire format. - * Reads hh/hl/lh/ll quads from off-heap (stored contiguously per value). - */ - private void writeDecimal256Column(byte scale, long addr, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - long offset = (long) i * 32; - buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset)); - buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 8)); - buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 16)); - buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 24)); - } - } - - /** - * Writes Decimal64 values in big-endian wire format. - * Reads longs from off-heap. - */ - private void writeDecimal64Column(byte scale, long addr, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + (long) i * 8)); - } - } - - private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - double[] data = col.getDoubleArrayData(); - - int shapeIdx = 0; - int dataIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - buffer.putByte((byte) nDims); - - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - int dimLen = shapes[shapeIdx++]; - buffer.putInt(dimLen); - elemCount = Math.multiplyExact(elemCount, dimLen); - } - - for (int e = 0; e < elemCount; e++) { - buffer.putDouble(data[dataIdx++]); - } - } - } - - /** - * Writes a GeoHash column in variable-width wire format. - *

    - * Wire format: [precision varint] [packed values: ceil(precision/8) bytes each] - * Values are stored as 8-byte longs in the off-heap buffer but only the - * lower ceil(precision/8) bytes are written to the wire. - */ - private void writeGeoHashColumn(long addr, int count, int precision) { - if (precision < 1) { - // All values are null: use minimum valid precision. - // The decoder will skip all values via the null bitmap, - // so the precision only needs to be structurally valid. - precision = 1; - } - buffer.putVarint(precision); - int valueSize = (precision + 7) / 8; - for (int i = 0; i < count; i++) { - long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8); - for (int b = 0; b < valueSize; b++) { - buffer.putByte((byte) (value >>> (b * 8))); - } - } - } - - private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - long[] data = col.getLongArrayData(); - - int shapeIdx = 0; - int dataIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - buffer.putByte((byte) nDims); - - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - int dimLen = shapes[shapeIdx++]; - buffer.putInt(dimLen); - elemCount = Math.multiplyExact(elemCount, dimLen); - } - - for (int e = 0; e < elemCount; e++) { - buffer.putLong(data[dataIdx++]); - } - } - } - - /** - * Writes a null bitmap from off-heap memory. - * On little-endian platforms, the byte layout of the long-packed bitmap - * in memory matches the wire format, enabling bulk copy. - */ - private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) { - long nullAddr = col.getNullBitmapAddress(); - if (nullAddr != 0) { - int bitmapSize = (rowCount + 7) / 8; - buffer.putBlockOfBytes(nullAddr, bitmapSize); - } else { - // Non-nullable column shouldn't reach here, but write zeros as fallback - int bitmapSize = (rowCount + 7) / 8; - for (int i = 0; i < bitmapSize; i++) { - buffer.putByte((byte) 0); - } - } - } - - private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) { - // Offset array: (valueCount + 1) int32 values, pre-built in wire format - buffer.putBlockOfBytes(col.getStringOffsetsAddress(), (long) (valueCount + 1) * 4); - // UTF-8 data: raw bytes, contiguous - buffer.putBlockOfBytes(col.getStringDataAddress(), col.getStringDataSize()); - } - - /** - * Writes a symbol column with dictionary. - * Reads local symbol indices from off-heap data buffer. - */ - private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) { - long dataAddr = col.getDataAddress(); - String[] dictionary = col.getSymbolDictionary(); - - buffer.putVarint(dictionary.length); - for (String symbol : dictionary) { - buffer.putString(symbol); - } - - for (int i = 0; i < count; i++) { - int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); - buffer.putVarint(idx); - } - } - - /** - * Writes a symbol column using global IDs (for delta dictionary mode). - * Reads from auxiliary data buffer if available, otherwise falls back to local indices. - */ - private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) { - long auxAddr = col.getAuxDataAddress(); - if (auxAddr == 0) { - // Fall back to local indices - long dataAddr = col.getDataAddress(); - for (int i = 0; i < count; i++) { - int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); - buffer.putVarint(idx); - } - } else { - for (int i = 0; i < count; i++) { - int globalId = Unsafe.getUnsafe().getInt(auxAddr + (long) i * 4); - buffer.putVarint(globalId); - } - } - } - - private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) { - buffer.putString(tableName); - buffer.putVarint(rowCount); - buffer.putVarint(columns.length); - buffer.putByte(SCHEMA_MODE_FULL); - for (QwpColumnDef col : columns) { - buffer.putString(col.getName()); - buffer.putByte(col.getWireTypeCode()); - } - } - - private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { - buffer.putString(tableName); - buffer.putVarint(rowCount); - buffer.putVarint(columnCount); - buffer.putByte(SCHEMA_MODE_REFERENCE); - buffer.putLong(schemaHash); - } - - /** - * Writes a timestamp column with optional Gorilla compression. - * Reads longs directly from off-heap — zero heap allocation. - */ - private void writeTimestampColumn(long addr, int count, boolean useGorilla) { - if (useGorilla && count > 2) { - if (QwpGorillaEncoder.canUseGorilla(addr, count)) { - buffer.putByte(ENCODING_GORILLA); - int encodedSize = QwpGorillaEncoder.calculateEncodedSize(addr, count); - buffer.ensureCapacity(encodedSize); - int bytesWritten = gorillaEncoder.encodeTimestamps( - buffer.getBufferPtr() + buffer.getPosition(), - buffer.getCapacity() - buffer.getPosition(), - addr, - count - ); - buffer.skip(bytesWritten); - } else { - buffer.putByte(ENCODING_UNCOMPRESSED); - buffer.putBlockOfBytes(addr, (long) count * 8); - } - } else { - if (useGorilla) { - buffer.putByte(ENCODING_UNCOMPRESSED); - } - buffer.putBlockOfBytes(addr, (long) count * 8); - } - } } From c49d75dfaf668c0d67980f0451f8d309ba550e2e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 5 Mar 2026 08:37:21 +0100 Subject: [PATCH 145/230] Magic bytes change fallout --- .../cutlass/qwp/client/NativeBufferWriterTest.java | 8 ++++---- .../qwp/client/QwpWebSocketEncoderTest.java | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index a2cda57..1214b45 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -26,10 +26,10 @@ import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Unsafe; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import static org.junit.Assert.*; public class NativeBufferWriterTest { @@ -66,10 +66,10 @@ public void testGrowBuffer() throws Exception { public void testMultipleWrites() throws Exception { assertMemoryLeak(() -> { try (NativeBufferWriter writer = new NativeBufferWriter()) { - writer.putByte((byte) 'I'); - writer.putByte((byte) 'L'); + writer.putByte((byte) 'Q'); + writer.putByte((byte) 'W'); writer.putByte((byte) 'P'); - writer.putByte((byte) '4'); + writer.putByte((byte) '1'); writer.putByte((byte) 1); // Version writer.putByte((byte) 0); // Flags writer.putShort((short) 1); // Table count diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index d5909c3..eab1c21 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -29,11 +29,11 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.Unsafe; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Test; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; /** * Unit tests for QwpWebSocketEncoder. @@ -446,10 +446,10 @@ public void testEncodeMultipleColumns() throws Exception { // Verify header QwpBufferWriter buf = encoder.getBuffer(); long ptr = buf.getBufferPtr(); - Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); - Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'Q', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'W', Unsafe.getUnsafe().getByte(ptr + 1)); Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); - Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + Assert.assertEquals((byte) '1', Unsafe.getUnsafe().getByte(ptr + 3)); } }); } @@ -692,10 +692,10 @@ public void testEncodeSingleRowWithLong() throws Exception { long ptr = buf.getBufferPtr(); // Verify header magic - Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); - Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'Q', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'W', Unsafe.getUnsafe().getByte(ptr + 1)); Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); - Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + Assert.assertEquals((byte) '1', Unsafe.getUnsafe().getByte(ptr + 3)); // Version Assert.assertEquals(VERSION_1, Unsafe.getUnsafe().getByte(ptr + 4)); From 8094d57b174886be16270e5a31871a503b35ab0f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 5 Mar 2026 10:35:55 +0100 Subject: [PATCH 146/230] Improve error message --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 22 ++++++++++--------- .../qwp/protocol/QwpTableBufferTest.java | 10 ++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index f3d7fe1..d6e3ea1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -169,7 +169,8 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) columnAccessCursor++; if (candidate.type != type) { throw new LineSenderException( - "Column type mismatch for " + name + ": existing=" + candidate.type + " new=" + type + "Column type mismatch for column '" + name + "': columnType=" + + candidate.type + ", sentType=" + type ); } return candidate; @@ -182,7 +183,8 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) ColumnBuffer existing = columns.get(idx); if (existing.type != type) { throw new LineSenderException( - "Column type mismatch for " + name + ": existing=" + existing.type + " new=" + type + "Column type mismatch for column '" + name + "': columnType=" + + existing.type + ", sentType=" + type ); } return existing; @@ -943,6 +945,14 @@ public int[] getArrayShapes() { return arrayShapes; } + /** + * Returns the off-heap address of the auxiliary data buffer (global symbol IDs). + * Returns 0 if no auxiliary data exists. + */ + public long getAuxDataAddress() { + return auxBuffer != null ? auxBuffer.pageAddress() : 0; + } + /** * Returns the total bytes buffered in this column's storage. */ @@ -969,14 +979,6 @@ public long getBufferedBytes() { return bytes; } - /** - * Returns the off-heap address of the auxiliary data buffer (global symbol IDs). - * Returns 0 if no auxiliary data exists. - */ - public long getAuxDataAddress() { - return auxBuffer != null ? auxBuffer.pageAddress() : 0; - } - /** * Returns the off-heap address of the column data buffer. */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index a1a7d0d..7f083c4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -30,12 +30,10 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal64; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import static org.junit.Assert.*; public class QwpTableBufferTest { @@ -420,7 +418,7 @@ public void testGetOrCreateColumnConflictingTypeFastPath() throws Exception { fail("Expected LineSenderException for column type mismatch"); } catch (LineSenderException e) { assertEquals( - "Column type mismatch for x: existing=" + QwpConstants.TYPE_LONG + " new=" + QwpConstants.TYPE_DOUBLE, + "Column type mismatch for column 'x': columnType=" + QwpConstants.TYPE_LONG + ", sentType=" + QwpConstants.TYPE_DOUBLE, e.getMessage() ); } @@ -445,7 +443,7 @@ public void testGetOrCreateColumnConflictingTypeSlowPath() throws Exception { fail("Expected LineSenderException for column type mismatch"); } catch (LineSenderException e) { assertEquals( - "Column type mismatch for b: existing=" + QwpConstants.TYPE_STRING + " new=" + QwpConstants.TYPE_LONG, + "Column type mismatch for column 'b': columnType=" + QwpConstants.TYPE_STRING + ", sentType=" + QwpConstants.TYPE_LONG, e.getMessage() ); } From b923ec2b1b8845dfdba8b2e266e999612af86d91 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 5 Mar 2026 11:49:51 +0100 Subject: [PATCH 147/230] update magic --- .../io/questdb/client/cutlass/qwp/client/QwpUdpSender.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index a82afc3..5363473 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -573,10 +573,10 @@ private void checkTableSelected() { private int encodeForUdp(QwpTableBuffer tableBuffer) { buffer.reset(); // Write 12-byte ILP4 header: magic, version, flags=0, tableCount=1, payloadLength=0 (patched later) - buffer.putByte((byte) 'I'); - buffer.putByte((byte) 'L'); + buffer.putByte((byte) 'Q'); + buffer.putByte((byte) 'W'); buffer.putByte((byte) 'P'); - buffer.putByte((byte) '4'); + buffer.putByte((byte) '1'); buffer.putByte(VERSION_1); buffer.putByte((byte) 0); // flags buffer.putShort((short) 1); // tableCount From fc2ace3ad45ea7c33fb94124dcf9e49727a5efad Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 5 Mar 2026 12:20:50 +0100 Subject: [PATCH 148/230] Add Sender.builder(WEBSOCKET) E2E test Add testSenderBuilderWebSocket() to QwpSenderTest which exercises the production Sender.builder(Transport.WEBSOCKET) path end-to-end. All existing E2E tests use QwpWebSocketSender.connect() directly, but production users would use Sender.builder(). The new test writes rows with mixed column types (symbol, double, long, boolean, string) and verifies data arrives correctly via SQL queries. Also convert string concatenation in assertion expected values to text blocks throughout QwpSenderTest for improved readability. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/QwpSenderTest.java | 1296 +++++++++++------ 1 file changed, 818 insertions(+), 478 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index 0d0d157..1629ce3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.Decimal128; @@ -76,9 +77,11 @@ public void testBoolToString() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + true\t1970-01-01T00:00:01.000000000Z + false\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -104,9 +107,11 @@ public void testBoolToVarchar() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + true\t1970-01-01T00:00:01.000000000Z + false\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -127,9 +132,11 @@ public void testBoolean() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "b\ttimestamp\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", + """ + b\ttimestamp + true\t1970-01-01T00:00:01.000000000Z + false\t1970-01-01T00:00:02.000000000Z + """, "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -546,9 +553,11 @@ public void testByteToDate() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -574,9 +583,11 @@ public void testByteToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -602,9 +613,11 @@ public void testByteToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -1.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -630,9 +643,11 @@ public void testByteToDecimal16() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -9.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -658,9 +673,11 @@ public void testByteToDecimal256() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -1.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -686,9 +703,11 @@ public void testByteToDecimal64() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -1.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -714,9 +733,11 @@ public void testByteToDecimal8() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 5.0\t1970-01-01T00:00:01.000000000Z + -9.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -742,9 +763,11 @@ public void testByteToDouble() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -770,9 +793,11 @@ public void testByteToFloat() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + f\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT f, ts FROM " + table + " ORDER BY ts"); } @@ -826,10 +851,12 @@ public void testByteToInt() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "127\t1970-01-01T00:00:02.000000000Z\n" + - "-128\t1970-01-01T00:00:03.000000000Z\n", + """ + i\tts + 42\t1970-01-01T00:00:01.000000000Z + 127\t1970-01-01T00:00:02.000000000Z + -128\t1970-01-01T00:00:03.000000000Z + """, "SELECT i, ts FROM " + table + " ORDER BY ts"); } @@ -858,10 +885,12 @@ public void testByteToLong() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "127\t1970-01-01T00:00:02.000000000Z\n" + - "-128\t1970-01-01T00:00:03.000000000Z\n", + """ + l\tts + 42\t1970-01-01T00:00:01.000000000Z + 127\t1970-01-01T00:00:02.000000000Z + -128\t1970-01-01T00:00:03.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -915,10 +944,12 @@ public void testByteToShort() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -128\t1970-01-01T00:00:02.000000000Z + 127\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -947,10 +978,12 @@ public void testByteToString() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + 0\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -979,10 +1012,12 @@ public void testByteToSymbol() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -1\t1970-01-01T00:00:02.000000000Z + 0\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -1008,9 +1043,11 @@ public void testByteToTimestamp() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + t\tts + 1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT t, ts FROM " + table + " ORDER BY ts"); } @@ -1064,10 +1101,12 @@ public void testByteToVarchar() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + """ + v\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + 127\t1970-01-01T00:00:03.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -1091,10 +1130,12 @@ public void testChar() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "c\ttimestamp\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "ü\t1970-01-01T00:00:02.000000000Z\n" + - "中\t1970-01-01T00:00:03.000000000Z\n", + """ + c\ttimestamp + A\t1970-01-01T00:00:01.000000000Z + ü\t1970-01-01T00:00:02.000000000Z + 中\t1970-01-01T00:00:03.000000000Z + """, "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -1330,9 +1371,11 @@ public void testCharToString() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "Z\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + A\t1970-01-01T00:00:01.000000000Z + Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -1400,9 +1443,11 @@ public void testCharToVarchar() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "Z\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + A\t1970-01-01T00:00:01.000000000Z + Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -1452,9 +1497,11 @@ public void testDecimal128ToDecimal256() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -1480,9 +1527,11 @@ public void testDecimal128ToDecimal64() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -1508,9 +1557,11 @@ public void testDecimal256ToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -1537,9 +1588,11 @@ public void testDecimal256ToDecimal64() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -1620,9 +1673,11 @@ public void testDecimal64ToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -1648,9 +1703,11 @@ public void testDecimal64ToDecimal256() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -1677,9 +1734,11 @@ public void testDecimalRescale() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.4500\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0000\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.4500\t1970-01-01T00:00:01.000000000Z + -1.0000\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -1936,9 +1995,11 @@ public void testDecimalToString() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -2048,9 +2109,11 @@ public void testDecimalToVarchar() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -2087,14 +2150,16 @@ public void testDouble() throws Exception { assertTableSizeEventually(table, 7); assertSqlEventually( - "d\ttimestamp\n" + - "42.5\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + - "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + - "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + - "null\t1970-01-01T00:00:05.000000000Z\n" + - "null\t1970-01-01T00:00:06.000000000Z\n" + - "null\t1970-01-01T00:00:07.000000000Z\n", + """ + d\ttimestamp + 42.5\t1970-01-01T00:00:01.000000000Z + -1.0E10\t1970-01-01T00:00:02.000000000Z + 1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z + 4.9E-324\t1970-01-01T00:00:04.000000000Z + null\t1970-01-01T00:00:05.000000000Z + null\t1970-01-01T00:00:06.000000000Z + null\t1970-01-01T00:00:07.000000000Z + """, "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -2246,9 +2311,11 @@ public void testDoubleToByte() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", + """ + b\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -2366,9 +2433,11 @@ public void testDoubleToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-42.10\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -42.10\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -2463,9 +2532,11 @@ public void testDoubleToInt() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "100000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", + """ + i\tts + 100000\t1970-01-01T00:00:01.000000000Z + -42\t1970-01-01T00:00:02.000000000Z + """, "SELECT i, ts FROM " + table + " ORDER BY ts"); } @@ -2516,9 +2587,11 @@ public void testDoubleToLong() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "1000000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", + """ + l\tts + 1000000\t1970-01-01T00:00:01.000000000Z + -42\t1970-01-01T00:00:02.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -2565,9 +2638,11 @@ public void testDoubleToShort() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "100\t1970-01-01T00:00:01.000000000Z\n" + - "-200\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + 100\t1970-01-01T00:00:01.000000000Z + -200\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -2593,9 +2668,11 @@ public void testDoubleToString() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + 3.14\t1970-01-01T00:00:01.000000000Z + -42.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -2618,8 +2695,10 @@ public void testDoubleToSymbol() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "sym\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n", + """ + sym\tts + 3.14\t1970-01-01T00:00:01.000000000Z + """, "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @@ -2666,9 +2745,11 @@ public void testDoubleToVarchar() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + 3.14\t1970-01-01T00:00:01.000000000Z + -42.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -2736,9 +2817,11 @@ public void testFloatToByte() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "7\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + 7\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -2806,9 +2889,11 @@ public void testFloatToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1.50\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1.50\t1970-01-01T00:00:01.000000000Z + -42.25\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -2859,9 +2944,11 @@ public void testFloatToDouble() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1.5\t1970-01-01T00:00:01.000000000Z + -42.25\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -2908,9 +2995,11 @@ public void testFloatToInt() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", + """ + i\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + """, "SELECT i, ts FROM " + table + " ORDER BY ts"); } @@ -2958,8 +3047,10 @@ public void testFloatToLong() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "l\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n", + """ + l\tts + 1000\t1970-01-01T00:00:01.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -3006,9 +3097,11 @@ public void testFloatToShort() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1000\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + 42\t1970-01-01T00:00:01.000000000Z + -1000\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -3031,8 +3124,10 @@ public void testFloatToString() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", + """ + s\tts + 1.5\t1970-01-01T00:00:01.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -3055,8 +3150,10 @@ public void testFloatToSymbol() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "sym\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", + """ + sym\tts + 1.5\t1970-01-01T00:00:01.000000000Z + """, "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @@ -3100,8 +3197,10 @@ public void testFloatToVarchar() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "v\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", + """ + v\tts + 1.5\t1970-01-01T00:00:01.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -3129,11 +3228,13 @@ public void testInt() throws Exception { assertTableSizeEventually(table, 4); assertSqlEventually( - "i\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n" + - "-42\t1970-01-01T00:00:04.000000000Z\n", + """ + i\ttimestamp + null\t1970-01-01T00:00:01.000000000Z + 0\t1970-01-01T00:00:02.000000000Z + 2147483647\t1970-01-01T00:00:03.000000000Z + -42\t1970-01-01T00:00:04.000000000Z + """, "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -3187,10 +3288,12 @@ public void testIntToByte() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + """ + b\tts + 42\t1970-01-01T00:00:01.000000000Z + -128\t1970-01-01T00:00:02.000000000Z + 127\t1970-01-01T00:00:03.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -3267,9 +3370,11 @@ public void testIntToDate() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3295,9 +3400,11 @@ public void testIntToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3326,10 +3433,12 @@ public void testIntToDecimal128() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + 0.00\t1970-01-01T00:00:03.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3358,10 +3467,12 @@ public void testIntToDecimal16() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + 0.0\t1970-01-01T00:00:03.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3390,10 +3501,12 @@ public void testIntToDecimal256() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + 0.00\t1970-01-01T00:00:03.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3422,10 +3535,12 @@ public void testIntToDecimal64() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + """ + d\tts + 2147483647.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + 0.00\t1970-01-01T00:00:03.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3454,10 +3569,12 @@ public void testIntToDecimal8() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + """ + d\tts + 5.0\t1970-01-01T00:00:01.000000000Z + -9.0\t1970-01-01T00:00:02.000000000Z + 0.0\t1970-01-01T00:00:03.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3483,9 +3600,11 @@ public void testIntToDouble() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -3514,10 +3633,12 @@ public void testIntToFloat() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + """ + f\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + 0.0\t1970-01-01T00:00:03.000000000Z + """, "SELECT f, ts FROM " + table + " ORDER BY ts"); } @@ -3571,10 +3692,12 @@ public void testIntToLong() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "2147483647\t1970-01-01T00:00:02.000000000Z\n" + - "-1\t1970-01-01T00:00:03.000000000Z\n", + """ + l\tts + 42\t1970-01-01T00:00:01.000000000Z + 2147483647\t1970-01-01T00:00:02.000000000Z + -1\t1970-01-01T00:00:03.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -3628,10 +3751,12 @@ public void testIntToShort() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 1000\t1970-01-01T00:00:01.000000000Z + -32768\t1970-01-01T00:00:02.000000000Z + 32767\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -3685,10 +3810,12 @@ public void testIntToString() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + 0\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -3717,10 +3844,12 @@ public void testIntToSymbol() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -1\t1970-01-01T00:00:02.000000000Z + 0\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -3747,9 +3876,11 @@ public void testIntToTimestamp() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + t\tts + 1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT t, ts FROM " + table + " ORDER BY ts"); } @@ -3803,10 +3934,12 @@ public void testIntToVarchar() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n", + """ + v\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + 2147483647\t1970-01-01T00:00:03.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -3831,10 +3964,12 @@ public void testLong() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "l\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + """ + l\ttimestamp + null\t1970-01-01T00:00:01.000000000Z + 0\t1970-01-01T00:00:02.000000000Z + 9223372036854775807\t1970-01-01T00:00:03.000000000Z + """, "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -4087,8 +4222,10 @@ public void testLong256ToString() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + """ + s\tts + 0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z + """, "SELECT s, ts FROM " + table); } @@ -4153,8 +4290,10 @@ public void testLong256ToVarchar() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "v\tts\n" + - "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + """ + v\tts + 0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z + """, "SELECT v, ts FROM " + table); } @@ -4208,10 +4347,12 @@ public void testLongToByte() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + """ + b\tts + 42\t1970-01-01T00:00:01.000000000Z + -128\t1970-01-01T00:00:02.000000000Z + 127\t1970-01-01T00:00:03.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -4287,9 +4428,11 @@ public void testLongToDate() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4315,9 +4458,11 @@ public void testLongToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4343,9 +4488,11 @@ public void testLongToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1000000000.00\t1970-01-01T00:00:01.000000000Z + -1000000000.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4371,9 +4518,11 @@ public void testLongToDecimal16() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4399,9 +4548,11 @@ public void testLongToDecimal256() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 9223372036854775807.00\t1970-01-01T00:00:01.000000000Z + -1000000000000.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4427,9 +4578,11 @@ public void testLongToDecimal32() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4455,9 +4608,11 @@ public void testLongToDecimal8() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 5.0\t1970-01-01T00:00:01.000000000Z + -9.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4483,9 +4638,11 @@ public void testLongToDouble() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4511,9 +4668,11 @@ public void testLongToFloat() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + f\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT f, ts FROM " + table + " ORDER BY ts"); } @@ -4565,9 +4724,11 @@ public void testLongToInt() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", + """ + i\tts + 42\t1970-01-01T00:00:01.000000000Z + -1\t1970-01-01T00:00:02.000000000Z + """, "SELECT i, ts FROM " + table + " ORDER BY ts"); } @@ -4692,9 +4853,11 @@ public void testLongToString() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + 9223372036854775807\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -4720,9 +4883,11 @@ public void testLongToSymbol() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -1\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -4748,9 +4913,11 @@ public void testLongToTimestamp() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + t\tts + 1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT t, ts FROM " + table + " ORDER BY ts"); } @@ -4801,9 +4968,11 @@ public void testLongToVarchar() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + 42\t1970-01-01T00:00:01.000000000Z + 9223372036854775807\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -4844,9 +5013,11 @@ public void testNullStringToBoolean() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", + """ + b\tts + true\t1970-01-01T00:00:01.000000000Z + false\t1970-01-01T00:00:02.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -4867,9 +5038,11 @@ public void testNullStringToByte() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n", + """ + b\tts + 42\t1970-01-01T00:00:01.000000000Z + 0\t1970-01-01T00:00:02.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -4890,9 +5063,11 @@ public void testNullStringToChar() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "c\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + c\tts + A\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT c, ts FROM " + table + " ORDER BY ts"); } @@ -4913,9 +5088,11 @@ public void testNullStringToDate() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4936,9 +5113,11 @@ public void testNullStringToDecimal() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -4959,9 +5138,11 @@ public void testNullStringToFloat() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + f\tts + 3.14\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT f, ts FROM " + table + " ORDER BY ts"); } @@ -4982,9 +5163,11 @@ public void testNullStringToGeoHash() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "g\tts\n" + - "s09wh\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + g\tts + s09wh\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT g, ts FROM " + table + " ORDER BY ts"); } @@ -5005,9 +5188,11 @@ public void testNullStringToLong256() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "0x01\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + l\tts + 0x01\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -5037,9 +5222,11 @@ public void testNullStringToNumeric() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tl\td\tts\n" + - "42\t100\t3.14\t1970-01-01T00:00:01.000000000Z\n" + - "null\tnull\tnull\t1970-01-01T00:00:02.000000000Z\n", + """ + i\tl\td\tts + 42\t100\t3.14\t1970-01-01T00:00:01.000000000Z + null\tnull\tnull\t1970-01-01T00:00:02.000000000Z + """, "SELECT i, l, d, ts FROM " + table + " ORDER BY ts"); } @@ -5060,9 +5247,11 @@ public void testNullStringToShort() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + 0\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -5083,9 +5272,11 @@ public void testNullStringToSymbol() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "alpha\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + alpha\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -5106,9 +5297,11 @@ public void testNullStringToTimestamp() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + t\tts + 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT t, ts FROM " + table + " ORDER BY ts"); } @@ -5129,9 +5322,11 @@ public void testNullStringToTimestampNs() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + t\tts + 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT t, ts FROM " + table + " ORDER BY ts"); } @@ -5152,9 +5347,11 @@ public void testNullStringToUuid() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "u\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + u\tts + a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT u, ts FROM " + table + " ORDER BY ts"); } @@ -5175,9 +5372,11 @@ public void testNullStringToVarchar() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + hello\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -5198,9 +5397,11 @@ public void testNullSymbolToString() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + hello\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -5221,9 +5422,11 @@ public void testNullSymbolToSymbol() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "alpha\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + alpha\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -5244,12 +5447,49 @@ public void testNullSymbolToVarchar() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + hello\t1970-01-01T00:00:01.000000000Z + null\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testSenderBuilderWebSocket() throws Exception { + String table = "test_qwp_sender_builder_ws"; + useTable(table); + + try (Sender sender = Sender.builder(Sender.Transport.WEBSOCKET) + .address(getQuestDbHost() + ":" + getHttpPort()) + .build()) { + sender.table(table) + .symbol("city", "London") + .doubleColumn("temp", 22.5) + .longColumn("humidity", 48) + .boolColumn("sunny", true) + .stringColumn("note", "clear sky") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("city", "Berlin") + .doubleColumn("temp", 18.3) + .longColumn("humidity", 65) + .boolColumn("sunny", false) + .stringColumn("note", "overcast") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + """ + city\ttemp\thumidity\tsunny\tnote\ttimestamp + London\t22.5\t48\ttrue\tclear sky\t1970-01-01T00:00:01.000000000Z + Berlin\t18.3\t65\tfalse\tovercast\t1970-01-01T00:00:02.000000000Z + """, + "SELECT city, temp, humidity, sunny, note, timestamp FROM " + table + " ORDER BY timestamp"); + } + @Test public void testShort() throws Exception { String table = "test_qwp_short"; @@ -5322,10 +5562,12 @@ public void testShortToByte() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + """ + b\tts + 42\t1970-01-01T00:00:01.000000000Z + -128\t1970-01-01T00:00:02.000000000Z + 127\t1970-01-01T00:00:03.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -5402,9 +5644,11 @@ public void testShortToDate() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5430,9 +5674,11 @@ public void testShortToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "32767.00\t1970-01-01T00:00:01.000000000Z\n" + - "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 32767.00\t1970-01-01T00:00:01.000000000Z + -32768.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5458,9 +5704,11 @@ public void testShortToDecimal16() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5486,9 +5734,11 @@ public void testShortToDecimal256() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5514,9 +5764,11 @@ public void testShortToDecimal32() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5542,9 +5794,11 @@ public void testShortToDecimal64() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.00\t1970-01-01T00:00:01.000000000Z + -100.00\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5570,9 +5824,11 @@ public void testShortToDecimal8() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 5.0\t1970-01-01T00:00:01.000000000Z + -9.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5598,9 +5854,11 @@ public void testShortToDouble() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -5626,9 +5884,11 @@ public void testShortToFloat() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + """ + f\tts + 42.0\t1970-01-01T00:00:01.000000000Z + -100.0\t1970-01-01T00:00:02.000000000Z + """, "SELECT f, ts FROM " + table + " ORDER BY ts"); } @@ -5679,9 +5939,11 @@ public void testShortToInt() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", + """ + i\tts + 42\t1970-01-01T00:00:01.000000000Z + 32767\t1970-01-01T00:00:02.000000000Z + """, "SELECT i, ts FROM " + table + " ORDER BY ts"); } @@ -5707,9 +5969,11 @@ public void testShortToLong() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", + """ + l\tts + 42\t1970-01-01T00:00:01.000000000Z + 32767\t1970-01-01T00:00:02.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -5763,10 +6027,12 @@ public void testShortToString() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + 0\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -5795,10 +6061,12 @@ public void testShortToSymbol() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 42\t1970-01-01T00:00:01.000000000Z + -1\t1970-01-01T00:00:02.000000000Z + 0\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -5824,9 +6092,11 @@ public void testShortToTimestamp() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + """ + t\tts + 1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z + 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z + """, "SELECT t, ts FROM " + table + " ORDER BY ts"); } @@ -5880,10 +6150,12 @@ public void testShortToVarchar() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", + """ + v\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + 32767\t1970-01-01T00:00:03.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -5910,11 +6182,13 @@ public void testString() throws Exception { assertTableSizeEventually(table, 4); assertSqlEventually( - "s\ttimestamp\n" + - "hello world\t1970-01-01T00:00:01.000000000Z\n" + - "non-ascii äöü\t1970-01-01T00:00:02.000000000Z\n" + - "\t1970-01-01T00:00:03.000000000Z\n" + - "null\t1970-01-01T00:00:04.000000000Z\n", + """ + s\ttimestamp + hello world\t1970-01-01T00:00:01.000000000Z + non-ascii äöü\t1970-01-01T00:00:02.000000000Z + \t1970-01-01T00:00:03.000000000Z + null\t1970-01-01T00:00:04.000000000Z + """, "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -5949,12 +6223,14 @@ public void testStringToBoolean() throws Exception { assertTableSizeEventually(table, 5); assertSqlEventually( - "b\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n" + - "true\t1970-01-01T00:00:03.000000000Z\n" + - "false\t1970-01-01T00:00:04.000000000Z\n" + - "true\t1970-01-01T00:00:05.000000000Z\n", + """ + b\tts + true\t1970-01-01T00:00:01.000000000Z + false\t1970-01-01T00:00:02.000000000Z + true\t1970-01-01T00:00:03.000000000Z + false\t1970-01-01T00:00:04.000000000Z + true\t1970-01-01T00:00:05.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -6008,10 +6284,12 @@ public void testStringToByte() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + """ + b\tts + 42\t1970-01-01T00:00:01.000000000Z + -128\t1970-01-01T00:00:02.000000000Z + 127\t1970-01-01T00:00:03.000000000Z + """, "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -6062,9 +6340,11 @@ public void testStringToChar() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "c\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "H\t1970-01-01T00:00:02.000000000Z\n", + """ + c\tts + A\t1970-01-01T00:00:01.000000000Z + H\t1970-01-01T00:00:02.000000000Z + """, "SELECT c, ts FROM " + table + " ORDER BY ts"); } @@ -6087,8 +6367,10 @@ public void testStringToDate() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "d\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + d\tts + 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6135,9 +6417,11 @@ public void testStringToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6163,9 +6447,11 @@ public void testStringToDecimal16() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "12.5\t1970-01-01T00:00:01.000000000Z\n" + - "-99.9\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 12.5\t1970-01-01T00:00:01.000000000Z + -99.9\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6191,9 +6477,11 @@ public void testStringToDecimal256() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6219,9 +6507,11 @@ public void testStringToDecimal32() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1234.56\t1970-01-01T00:00:01.000000000Z\n" + - "-999.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1234.56\t1970-01-01T00:00:01.000000000Z + -999.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6247,9 +6537,11 @@ public void testStringToDecimal64() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 123.45\t1970-01-01T00:00:01.000000000Z + -99.99\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6275,9 +6567,11 @@ public void testStringToDecimal8() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-9.9\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 1.5\t1970-01-01T00:00:01.000000000Z + -9.9\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6303,9 +6597,11 @@ public void testStringToDouble() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-2.718\t1970-01-01T00:00:02.000000000Z\n", + """ + d\tts + 3.14\t1970-01-01T00:00:01.000000000Z + -2.718\t1970-01-01T00:00:02.000000000Z + """, "SELECT d, ts FROM " + table + " ORDER BY ts"); } @@ -6352,9 +6648,11 @@ public void testStringToFloat() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-2.5\t1970-01-01T00:00:02.000000000Z\n", + """ + f\tts + 3.14\t1970-01-01T00:00:01.000000000Z + -2.5\t1970-01-01T00:00:02.000000000Z + """, "SELECT f, ts FROM " + table + " ORDER BY ts"); } @@ -6401,9 +6699,11 @@ public void testStringToGeoHash() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "g\tts\n" + - "s24se\t1970-01-01T00:00:01.000000000Z\n" + - "u33dc\t1970-01-01T00:00:02.000000000Z\n", + """ + g\tts + s24se\t1970-01-01T00:00:01.000000000Z + u33dc\t1970-01-01T00:00:02.000000000Z + """, "SELECT g, ts FROM " + table + " ORDER BY ts"); } @@ -6453,10 +6753,12 @@ public void testStringToInt() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + """ + i\tts + 42\t1970-01-01T00:00:01.000000000Z + -100\t1970-01-01T00:00:02.000000000Z + 0\t1970-01-01T00:00:03.000000000Z + """, "SELECT i, ts FROM " + table + " ORDER BY ts"); } @@ -6503,9 +6805,11 @@ public void testStringToLong() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "1000000000000\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", + """ + l\tts + 1000000000000\t1970-01-01T00:00:01.000000000Z + -1\t1970-01-01T00:00:02.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -6528,8 +6832,10 @@ public void testStringToLong256() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "l\tts\n" + - "0x01\t1970-01-01T00:00:01.000000000Z\n", + """ + l\tts + 0x01\t1970-01-01T00:00:01.000000000Z + """, "SELECT l, ts FROM " + table + " ORDER BY ts"); } @@ -6600,10 +6906,12 @@ public void testStringToShort() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", + """ + s\tts + 1000\t1970-01-01T00:00:01.000000000Z + -32768\t1970-01-01T00:00:02.000000000Z + 32767\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -6650,9 +6958,11 @@ public void testStringToSymbol() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + hello\t1970-01-01T00:00:01.000000000Z + world\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -6675,8 +6985,10 @@ public void testStringToTimestamp() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "t\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + t\tts + 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT t, ts FROM " + table + " ORDER BY ts"); } @@ -6699,8 +7011,10 @@ public void testStringToTimestampNs() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + ts_col\tts + 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT ts_col, ts FROM " + table); } @@ -6744,8 +7058,10 @@ public void testStringToUuid() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "u\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + """ + u\tts + a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z + """, "SELECT u, ts FROM " + table + " ORDER BY ts"); } @@ -6787,9 +7103,11 @@ public void testStringToVarchar() throws Exception { } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + hello\t1970-01-01T00:00:01.000000000Z + world\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -6814,10 +7132,12 @@ public void testSymbol() throws Exception { assertTableSizeEventually(table, 3); assertSqlEventually( - "s\ttimestamp\n" + - "alpha\t1970-01-01T00:00:01.000000000Z\n" + - "beta\t1970-01-01T00:00:02.000000000Z\n" + - "alpha\t1970-01-01T00:00:03.000000000Z\n", + """ + s\ttimestamp + alpha\t1970-01-01T00:00:01.000000000Z + beta\t1970-01-01T00:00:02.000000000Z + alpha\t1970-01-01T00:00:03.000000000Z + """, "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -7095,9 +7415,11 @@ public void testSymbolToString() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", + """ + s\tts + hello\t1970-01-01T00:00:01.000000000Z + world\t1970-01-01T00:00:02.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -7186,9 +7508,11 @@ public void testSymbolToVarchar() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", + """ + v\tts + hello\t1970-01-01T00:00:01.000000000Z + world\t1970-01-01T00:00:02.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -7207,8 +7531,10 @@ public void testTimestampMicros() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "ts_col\ttimestamp\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + ts_col\ttimestamp + 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT ts_col, timestamp FROM " + table); } @@ -7233,8 +7559,10 @@ public void testTimestampMicrosToNanos() throws Exception { assertTableSizeEventually(table, 1); // Microseconds scaled to nanoseconds assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + ts_col\tts + 2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT ts_col, ts FROM " + table); } @@ -7275,8 +7603,10 @@ public void testTimestampNanosToMicros() throws Exception { assertTableSizeEventually(table, 1); // Nanoseconds truncated to microseconds assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + ts_col\tts + 2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT ts_col, ts FROM " + table); } @@ -7552,8 +7882,10 @@ public void testTimestampToString() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + s\tts + 2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT s, ts FROM " + table + " ORDER BY ts"); } @@ -7619,8 +7951,10 @@ public void testTimestampToVarchar() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "v\tts\n" + - "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + """ + v\tts + 2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z + """, "SELECT v, ts FROM " + table + " ORDER BY ts"); } @@ -7644,9 +7978,11 @@ public void testUuid() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( - "u\ttimestamp\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + - "11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z\n", + """ + u\ttimestamp + a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z + 11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z + """, "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -7916,8 +8252,10 @@ public void testUuidToString() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + """ + s\tts + a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z + """, "SELECT s, ts FROM " + table); } @@ -7963,8 +8301,10 @@ public void testUuidToVarchar() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( - "v\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + """ + v\tts + a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z + """, "SELECT v, ts FROM " + table); } From 6739bc0f940cebe7a8f18b39830fe81d1be883ad Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 5 Mar 2026 13:54:13 +0100 Subject: [PATCH 149/230] Validate table and column names in QwpWebSocketSender QwpWebSocketSender.table() accepted any CharSequence without validation, unlike the HTTP sender which validates via TableUtils. Empty, null, or whitespace-only table names passed through to the server, causing internal errors instead of clear client-side diagnostics. Add validateTableName() in table() and checkedColumnName() in all column methods. Both use TableUtils.isValidTableName/ isValidColumnName to reject empty, null, too-long, and names with illegal characters, throwing LineSenderException with a descriptive message. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 82 +++++++++++++------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3b77d73..cdfbe06 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client; import io.questdb.client.Sender; +import io.questdb.client.cairo.TableUtils; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketClientFactory; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; @@ -110,6 +111,7 @@ public class QwpWebSocketSender implements Sender { public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 private static final int DEFAULT_BUFFER_SIZE = 8192; + private static final int DEFAULT_MAX_NAME_LENGTH = 127; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final String WRITE_PATH = "/write/v4"; @@ -386,7 +388,7 @@ public void atNow() { public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_BOOLEAN, false); col.addBoolean(value); return this; } @@ -406,7 +408,7 @@ public DirectByteSlice bufferView() { public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_BYTE, false); col.addByte(value); return this; } @@ -431,7 +433,7 @@ public void cancelRow() { public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_CHAR, false); col.addShort((short) value); return this; } @@ -512,7 +514,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL64, true); col.addDecimal64(value); return this; } @@ -522,7 +524,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL128, true); col.addDecimal128(value); return this; } @@ -532,7 +534,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL256, true); col.addDecimal256(value); return this; } @@ -544,7 +546,7 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { checkTableSelected(); try { currentDecimal256.ofString(value); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL256, true); col.addDecimal256(currentDecimal256); } catch (Exception e) { throw new LineSenderException("Failed to parse decimal value: " + value, e); @@ -557,7 +559,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -567,7 +569,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -577,7 +579,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -587,7 +589,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(array); return this; } @@ -596,7 +598,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_DOUBLE, false); col.addDouble(value); return this; } @@ -611,7 +613,7 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_FLOAT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_FLOAT, false); col.addFloat(value); return this; } @@ -706,7 +708,7 @@ public QwpTableBuffer getTableBuffer(String tableName) { public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_INT, false); col.addInt(value); return this; } @@ -731,7 +733,7 @@ public boolean isGorillaEnabled() { public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG256, true); col.addLong256(l0, l1, l2, l3); return this; } @@ -741,7 +743,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -751,7 +753,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -761,7 +763,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -771,7 +773,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); col.addLongArray(array); return this; } @@ -780,7 +782,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG, false); col.addLong(value); return this; } @@ -822,7 +824,7 @@ public void setGorillaEnabled(boolean enabled) { public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_SHORT, false); col.addShort(value); return this; } @@ -831,7 +833,7 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_STRING, true); col.addString(value != null ? value.toString() : null); return this; } @@ -840,7 +842,7 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_SYMBOL, true); if (value != null) { // Register symbol in global dictionary and track max ID for delta calculation @@ -860,6 +862,7 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { @Override public QwpWebSocketSender table(CharSequence tableName) { checkNotClosed(); + validateTableName(tableName); // Fast path: if table name matches current, skip hashmap lookup if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { return this; @@ -882,11 +885,11 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkNotClosed(); checkTableSelected(); if (unit == ChronoUnit.NANOS) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP_NANOS, true); col.addLong(value); } else { long micros = toMicros(value, unit); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP, true); col.addLong(micros); } return this; @@ -897,7 +900,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkNotClosed(); checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP, true); col.addLong(micros); return this; } @@ -913,7 +916,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_UUID, true); col.addUuid(hi, lo); return this; } @@ -950,6 +953,19 @@ private void checkTableSelected() { } } + private String checkedColumnName(CharSequence name) { + if (name == null || !TableUtils.isValidColumnName(name, DEFAULT_MAX_NAME_LENGTH)) { + if (name == null || name.length() == 0) { + throw new LineSenderException("column name cannot be empty"); + } + if (name.length() > DEFAULT_MAX_NAME_LENGTH) { + throw new LineSenderException("column name too long [maxLength=" + DEFAULT_MAX_NAME_LENGTH + "]"); + } + throw new LineSenderException("column name contains illegal characters: " + name); + } + return name.toString(); + } + /** * Ensures the active buffer is ready for writing (in FILLING state). * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. @@ -1314,6 +1330,18 @@ private long toMicros(long value, ChronoUnit unit) { }; } + private void validateTableName(CharSequence name) { + if (name == null || !TableUtils.isValidTableName(name, DEFAULT_MAX_NAME_LENGTH)) { + if (name == null || name.length() == 0) { + throw new LineSenderException("table name cannot be empty"); + } + if (name.length() > DEFAULT_MAX_NAME_LENGTH) { + throw new LineSenderException("table name too long [maxLength=" + DEFAULT_MAX_NAME_LENGTH + "]"); + } + throw new LineSenderException("table name contains illegal characters: " + name); + } + } + /** * Waits synchronously for an ACK from the server for the specified batch. */ From 4fa6e6870593b1431d453c9595f126380525994a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 5 Mar 2026 14:15:23 +0100 Subject: [PATCH 150/230] Wrap NumericException on decimal rescale QwpTableBuffer.addDecimal64/128/256 silently rescale values when scales differ, but when rescaling down would lose precision, Decimal256.rescale() threw a raw NumericException that leaked to the caller unwrapped. Catch NumericException from rescale() and wrap it in LineSenderException with a clear message naming the column and the incompatible scales. Add unit tests for the Decimal64 and Decimal128 precision-loss paths. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 22 ++++++++-- .../qwp/protocol/QwpTableBufferTest.java | 40 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index d6e3ea1..43589ad 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -34,6 +34,7 @@ import io.questdb.client.std.Decimal64; import io.questdb.client.std.Decimals; import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.NumericException; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; @@ -471,7 +472,12 @@ public void addDecimal128(Decimal128 value) { } else if (decimalScale != value.getScale()) { rescaleTemp.ofRaw(value.getHigh(), value.getLow()); rescaleTemp.setScale(value.getScale()); - rescaleTemp.rescale(decimalScale); + try { + rescaleTemp.rescale(decimalScale); + } catch (NumericException e) { + throw new LineSenderException("column '" + name + "' cannot rescale decimal from scale " + + value.getScale() + " to " + decimalScale + " without precision loss", e); + } if (!rescaleTemp.fitsInStorageSizePow2(4)) { throw new LineSenderException("Decimal128 overflow: rescaling from scale " + value.getScale() + " to " + decimalScale + " exceeds 128-bit capacity"); @@ -499,7 +505,12 @@ public void addDecimal256(Decimal256 value) { decimalScale = (byte) value.getScale(); } else if (decimalScale != value.getScale()) { rescaleTemp.copyFrom(value); - rescaleTemp.rescale(decimalScale); + try { + rescaleTemp.rescale(decimalScale); + } catch (NumericException e) { + throw new LineSenderException("column '" + name + "' cannot rescale decimal from scale " + + value.getScale() + " to " + decimalScale + " without precision loss", e); + } src = rescaleTemp; } dataBuffer.putLong(src.getHh()); @@ -522,7 +533,12 @@ public void addDecimal64(Decimal64 value) { } else if (decimalScale != value.getScale()) { rescaleTemp.ofRaw(value.getValue()); rescaleTemp.setScale(value.getScale()); - rescaleTemp.rescale(decimalScale); + try { + rescaleTemp.rescale(decimalScale); + } catch (NumericException e) { + throw new LineSenderException("column '" + name + "' cannot rescale decimal from scale " + + value.getScale() + " to " + decimalScale + " without precision loss", e); + } if (!rescaleTemp.fitsInStorageSizePow2(3)) { throw new LineSenderException("Decimal64 overflow: rescaling from scale " + value.getScale() + " to " + decimalScale + " exceeds 64-bit capacity"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 7f083c4..f690b98 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -37,6 +37,26 @@ public class QwpTableBufferTest { + @Test + public void testAddDecimal128PrecisionLoss() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL128, true); + // First row sets decimalScale = 2 + col.addDecimal128(Decimal128.fromLong(100, 2)); + table.nextRow(); + // Second row at scale 4 with trailing fractional digits that + // cannot be represented at scale 2 without rounding + try { + col.addDecimal128(Decimal128.fromLong(12345, 4)); + fail("Expected LineSenderException for precision loss"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("precision loss")); + } + } + }); + } + @Test public void testAddDecimal128RescaleOverflow() throws Exception { assertMemoryLeak(() -> { @@ -57,6 +77,26 @@ public void testAddDecimal128RescaleOverflow() throws Exception { }); } + @Test + public void testAddDecimal64PrecisionLoss() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + // First row sets decimalScale = 2 + col.addDecimal64(Decimal64.fromLong(100, 2)); + table.nextRow(); + // Second row at scale 4 with trailing fractional digits that + // cannot be represented at scale 2 without rounding + try { + col.addDecimal64(Decimal64.fromLong(12345, 4)); + fail("Expected LineSenderException for precision loss"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("precision loss")); + } + } + }); + } + @Test public void testAddDecimal64RescaleOverflow() throws Exception { assertMemoryLeak(() -> { From e9111bd505b49262557ae897bcee1601ac8de472 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 5 Mar 2026 16:40:12 +0100 Subject: [PATCH 151/230] Handle edge cases for canceling and replaying rows. Added logic to handle rows that exceed size limits after replay and properly reset column states for late-added columns. --- .../cutlass/qwp/client/QwpUdpSender.java | 9 ++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 28 +++- .../qwp/protocol/QwpTableBufferTest.java | 153 ++++++++++++++++++ 3 files changed, 187 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 5363473..f229af0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -634,6 +634,15 @@ private void maybeAutoFlush() { currentTableBuffer.cancelCurrentRow(); flushSingleTable(currentTableName, currentTableBuffer); replayRowJournal(); + // Post-replay check: the replayed row alone may still exceed the limit. + tentativeRowCount = currentTableBuffer.getRowCount() + 1; + estimate = QwpDatagramSizeEstimator.estimate(currentTableBuffer, tentativeRowCount); + if (estimate > maxDatagramSize) { + throw new LineSenderException( + "single row exceeds maximum datagram size (" + maxDatagramSize + + " bytes), estimated " + estimate + " bytes" + ); + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index d6e3ea1..ffca0cc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -58,6 +58,7 @@ public class QwpTableBuffer implements QuietCloseable { private QwpColumnDef[] cachedColumnDefs; private int columnAccessCursor; // tracks expected next column index private boolean columnDefsCacheValid; + private int committedColumnCount; // columns that existed at last nextRow() private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access private int rowCount; private long schemaHash; @@ -80,12 +81,18 @@ public QwpTableBuffer(String tableName) { * If no values have been added for the current row, this is a no-op. */ public void cancelCurrentRow() { - // Reset sequential access cursor columnAccessCursor = 0; - // Truncate each column back to the committed row count for (int i = 0, n = columns.size(); i < n; i++) { ColumnBuffer col = fastColumns[i]; - col.truncateTo(rowCount); + if (i >= committedColumnCount) { + // Column was created during the in-progress row. Remove all data. + col.truncateTo(0); + } else if (col.size > rowCount) { + // Pre-existing column was set for the in-progress row. + // Truncate to committed state. + col.truncateTo(rowCount); + } + // else: pre-existing column wasn't touched this row. No-op. } } @@ -101,6 +108,7 @@ public void clear() { columnNameToIndex.clear(); fastColumns = null; columnAccessCursor = 0; + committedColumnCount = 0; rowCount = 0; schemaHash = 0; schemaHashComputed = false; @@ -257,6 +265,7 @@ public void nextRow() { } } rowCount++; + committedColumnCount = columns.size(); } /** @@ -267,6 +276,7 @@ public void reset() { fastColumns[i].reset(); } columnAccessCursor = 0; + committedColumnCount = columns.size(); rowCount = 0; } @@ -1157,6 +1167,18 @@ public void truncateTo(int newSize) { arrayShapeOffset = newShapeOffset; arrayDataOffset = newDataOffset; } + + // When all values are removed, reset type-specific metadata so the + // column behaves as freshly created (matches what reset() does). + if (newValueCount == 0) { + decimalScale = -1; + geohashPrecision = -1; + maxGlobalSymbolId = -1; + if (symbolDict != null) { + symbolDict.clear(); + symbolList.clear(); + } + } } private static int checkedElementCount(long product) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 7f083c4..9e6c854 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -177,6 +177,159 @@ public void testAddSymbolNullOnNonNullableColumn() throws Exception { }); } + @Test + public void testCancelRowTruncatesLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Commit 3 rows with columns "a" (LONG, non-nullable) and "b" (STRING, nullable) + for (int i = 0; i < 3; i++) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(i); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v" + i); + table.nextRow(); + } + + // Start row 4: set "a" and "b", then create a NEW column "c" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); + colC.addString("stale"); + + // Cancel the in-progress row + table.cancelCurrentRow(); + + // Column "c" was created during the in-progress row, so it must be fully cleared + assertEquals(0, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // Start row 4 again: set "a" and "b" only (not "c") + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); + table.nextRow(); + + // Column "c" should now have size == 4 (padded with nulls) and valueCount == 0 + assertEquals(4, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // All 4 rows of column "c" should be null + for (int i = 0; i < 4; i++) { + assertTrue("row " + i + " of column c should be null", colC.isNull(i)); + } + } + }); + } + + @Test + public void testCancelRowTruncatesLateAddedColumnWhenSizeEqualsRowCount() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Commit exactly 1 row so rowCount == 1 + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: set "a", then create NEW column "c" with one value + // col_c.size will be 1, which equals rowCount — the edge case + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); + colC.addString("stale"); + + // Cancel the in-progress row + table.cancelCurrentRow(); + + // Column "c" had size == rowCount (1 == 1) but was still late-added + assertEquals(0, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // Start row 2 again without setting "c" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + table.nextRow(); + + // Column "c" should have 2 null rows + assertEquals(2, colC.getSize()); + assertEquals(0, colC.getValueCount()); + assertTrue(colC.isNull(0)); + assertTrue(colC.isNull(1)); + } + }); + } + + @Test + public void testCancelRowResetsDecimalScaleOnLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: create a decimal column with scale 5 + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colD = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + colD.addDecimal64(Decimal64.fromLong(100, 5)); + table.cancelCurrentRow(); + + // After cancel, decimalScale must be reset. Adding a value at scale 3 + // should succeed and use scale 3 as the column's scale. + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colD.addDecimal64(Decimal64.fromLong(42, 3)); + table.nextRow(); + + assertEquals(2, colD.getSize()); + assertEquals(1, colD.getValueCount()); + } + }); + } + + @Test + public void testCancelRowResetsGeohashPrecisionOnLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: create a geohash column with 20-bit precision + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colG = table.getOrCreateColumn("g", QwpConstants.TYPE_GEOHASH, true); + colG.addGeoHash(123L, 20); + table.cancelCurrentRow(); + + // After cancel, geohashPrecision must be reset. Adding a value at + // 30-bit precision should succeed without a precision mismatch error. + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colG.addGeoHash(456L, 30); + table.nextRow(); + + assertEquals(2, colG.getSize()); + assertEquals(1, colG.getValueCount()); + } + }); + } + + @Test + public void testCancelRowResetsSymbolDictOnLateAddedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: create a symbol column with value "stale" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colS = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true); + colS.addSymbol("stale"); + table.cancelCurrentRow(); + + // After cancel, symbol dictionary must be empty. + // "fresh" should get local ID 0, not 1. + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colS.addSymbol("fresh"); + table.nextRow(); + + assertEquals(2, colS.getSize()); + assertEquals(1, colS.getValueCount()); + String[] dict = colS.getSymbolDictionary(); + assertEquals(1, dict.length); + assertEquals("fresh", dict[0]); + } + }); + } + @Test public void testCancelRowRewindsDoubleArrayOffsets() throws Exception { assertMemoryLeak(() -> { From 0c20920b8a5e3928054399841f612c724a0f8fa9 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 5 Mar 2026 16:57:20 +0100 Subject: [PATCH 152/230] reduce allocations --- .../cutlass/qwp/client/QwpUdpSender.java | 106 +++++++----------- .../qwp/protocol/OffHeapAppendMemory.java | 2 +- .../cutlass/qwp/protocol/QwpTableBuffer.java | 13 ++- 3 files changed, 51 insertions(+), 70 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index f229af0..120af2a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -134,13 +134,12 @@ public void atNow() { public Sender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - String name = columnName.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_BOOLEAN, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_BOOLEAN, false); col.addBoolean(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_BOOL; - e.name = name; + e.name = col.getName(); e.boolValue = value; } return this; @@ -190,13 +189,12 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DECIMAL64, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL64, true); col.addDecimal64(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL64; - e.name = colName; + e.name = col.getName(); e.objectValue = value; } return this; @@ -207,13 +205,12 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DECIMAL128, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL128, true); col.addDecimal128(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL128; - e.name = colName; + e.name = col.getName(); e.objectValue = value; } return this; @@ -224,13 +221,12 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL256, true); col.addDecimal256(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL256; - e.name = colName; + e.name = col.getName(); e.objectValue = value; } return this; @@ -241,13 +237,12 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = values; } return this; @@ -258,13 +253,12 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = values; } return this; @@ -275,13 +269,12 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = values; } return this; @@ -292,13 +285,12 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(array); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = array; } return this; @@ -308,13 +300,12 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public Sender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - String name = columnName.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_DOUBLE, false); col.addDouble(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE; - e.name = name; + e.name = col.getName(); e.doubleValue = value; } return this; @@ -331,13 +322,12 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); col.addLongArray(values); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = values; } return this; @@ -348,13 +338,12 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); col.addLongArray(values); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = values; } return this; @@ -365,13 +354,12 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); col.addLongArray(values); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = values; } return this; @@ -382,13 +370,12 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - String colName = name.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(colName, TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); col.addLongArray(array); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG_ARRAY; - e.name = colName; + e.name = col.getName(); e.objectValue = array; } return this; @@ -398,13 +385,12 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public Sender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - String name = columnName.toString(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_LONG, false); col.addLong(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG; - e.name = name; + e.name = col.getName(); e.longValue = value; } return this; @@ -431,15 +417,13 @@ public void reset() { public Sender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - String name = columnName.toString(); - String strValue = value != null ? value.toString() : null; - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_STRING, true); - col.addString(strValue); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_STRING, true); + col.addString(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_STRING; - e.name = name; - e.stringValue = strValue; + e.name = col.getName(); + e.stringValue = Chars.toString(value); // todo: allocation! } return this; } @@ -448,15 +432,13 @@ public Sender stringColumn(CharSequence columnName, CharSequence value) { public Sender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - String name = columnName.toString(); - String strValue = value != null ? value.toString() : null; - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_SYMBOL, true); - col.addSymbol(strValue); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_SYMBOL, true); + col.addSymbol(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_SYMBOL; - e.name = name; - e.stringValue = strValue; + e.name = col.getName(); + e.stringValue = Chars.toString(value); // todo: allocation! } return this; } @@ -474,7 +456,7 @@ public Sender table(CharSequence tableName) { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; rowJournalSize = 0; - currentTableName = tableName.toString(); + currentTableName = tableName.toString(); // todo: allocation! currentTableBuffer = tableBuffers.get(currentTableName); if (currentTableBuffer == null) { currentTableBuffer = new QwpTableBuffer(currentTableName); @@ -487,24 +469,23 @@ public Sender table(CharSequence tableName) { public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); - String name = columnName.toString(); if (unit == ChronoUnit.NANOS) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_TIMESTAMP_NANOS, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP_NANOS, true); col.addLong(value); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_TIMESTAMP_COL_NANOS; - e.name = name; + e.name = col.getName(); e.longValue = value; } } else { long micros = toMicros(value, unit); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); col.addLong(micros); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_TIMESTAMP_COL_MICROS; - e.name = name; + e.name = col.getName(); e.longValue = micros; } } @@ -515,14 +496,13 @@ public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit un public Sender timestampColumn(CharSequence columnName, Instant value) { checkNotClosed(); checkTableSelected(); - String name = columnName.toString(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); col.addLong(micros); if (maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_TIMESTAMP_COL_MICROS; - e.name = name; + e.name = col.getName(); e.longValue = micros; } return this; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index a30cf3c..4d418f4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -138,7 +138,7 @@ public void putShort(short value) { * Encodes a Java String to UTF-8 directly into the off-heap buffer. * Pre-ensures worst-case capacity to avoid per-byte checks. */ - public void putUtf8(String value) { + public void putUtf8(CharSequence value) { if (value == null || value.isEmpty()) { return; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 77c4ee0..5cc8959 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.std.CharSequenceIntHashMap; +import io.questdb.client.std.Chars; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -169,12 +170,12 @@ public QwpColumnDef[] getColumnDefs() { * Optimized for the common case where columns are accessed in the same * order every row: a sequential cursor avoids hash map lookups entirely. */ - public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) { + public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean nullable) { // Fast path: predict next column in sequence int n = columns.size(); if (columnAccessCursor < n) { ColumnBuffer candidate = fastColumns[columnAccessCursor]; - if (candidate.name.equals(name)) { + if (Chars.equals(candidate.name, name)) { columnAccessCursor++; if (candidate.type != type) { throw new LineSenderException( @@ -200,7 +201,7 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) } // Create new column - ColumnBuffer col = new ColumnBuffer(name, type, nullable); + ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); int index = columns.size(); columns.add(col); columnNameToIndex.put(name, index); @@ -870,7 +871,7 @@ public void addShort(short value) { size++; } - public void addString(String value) { + public void addString(CharSequence value) { if (value == null && nullable) { ensureNullCapacity(size + 1); markNull(size); @@ -885,7 +886,7 @@ public void addString(String value) { size++; } - public void addSymbol(String value) { + public void addSymbol(CharSequence value) { if (value == null) { addNull(); return; @@ -895,7 +896,7 @@ public void addSymbol(String value) { if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { idx = symbolList.size(); symbolDict.put(value, idx); - symbolList.add(value); + symbolList.add(Chars.toString(value)); } dataBuffer.putInt(idx); valueCount++; From 3de3e2305109238cdac7917479d7bdf05ef0bf00 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 10:56:10 +0100 Subject: [PATCH 153/230] delta datagram size estimation --- TODO.md | 32 + .../qwp/client/NativeBufferWriter.java | 11 + .../qwp/client/QwpDatagramSizeEstimator.java | 215 --- .../cutlass/qwp/client/QwpUdpSender.java | 926 ++++++++---- .../cutlass/qwp/protocol/QwpTableBuffer.java | 21 + .../qwp/client/NativeBufferWriterTest.java | 14 + .../client/QwpDatagramSizeEstimatorTest.java | 564 -------- .../cutlass/qwp/client/QwpUdpSenderTest.java | 1254 +++++++++++++++++ 8 files changed, 1989 insertions(+), 1048 deletions(-) create mode 100644 TODO.md delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..e7eecd2 --- /dev/null +++ b/TODO.md @@ -0,0 +1,32 @@ +# TODO + +## QWP UDP Sender + +### Documented limitation: mixing `atNow()` with `atMicros()` / `atNanos()` + +Current behavior in `QwpUdpSender` is to reject this pattern once committed rows already exist for the table: + +1. Write row(s) with `atNow()` (server-assigned designated timestamp). +2. Start a later row and finish it with `atMicros()` or `atNanos()`. + +The sender throws: + +- `schema change in middle of row is not supported` + +Why this happens: + +- `atNow()` does not write the designated timestamp column. +- `atMicros()` / `atNanos()` writes designated timestamp into the empty-name column (`""`). +- With committed rows already present, introducing this column is treated as schema evolution. +- The UDP incremental-estimate policy forbids schema changes in the middle of an in-progress row. + +Current workaround: + +- Use one designated timestamp strategy consistently per table stream: + - always `atNow()`, or + - always `atMicros()` / `atNanos()`. + +Future fix options: + +- Add explicit support for switching designated timestamp strategy mid-stream by pre-materializing designated timestamp schema state, or +- Harmonize designated timestamp handling so `atNow()` and `atMicros()` / `atNanos()` do not diverge schema shape. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index e538769..0f44bc1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -81,6 +81,17 @@ public static int utf8Length(String s) { return len; } + /** + * Returns the number of bytes required to encode {@code value} as an + * unsigned LEB128 varint. + */ + public static int varintSize(long value) { + if (value == 0) { + return 1; + } + return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; + } + @Override public void close() { if (bufferPtr != 0) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java deleted file mode 100644 index 4d5ec39..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDatagramSizeEstimator.java +++ /dev/null @@ -1,215 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.client; - -import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; -import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; - -import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; - -/** - * Estimates the encoded datagram size for a {@link QwpTableBuffer}. - *

    - * The estimate mirrors the encoding in {@link QwpWebSocketEncoder#encode} - * with {@code useSchemaRef=false} and Gorilla disabled. The estimate is - * always >= the actual encoded size. - */ -public final class QwpDatagramSizeEstimator { - - private QwpDatagramSizeEstimator() { - } - - /** - * Estimates the encoded datagram size in bytes. - *

    - * The {@code rowCount} parameter is separate from {@code tableBuffer.getRowCount()} - * so the caller can pass {@code rowCount + 1} for a tentative pre-commit estimate. - * - * @param tableBuffer the table buffer to estimate - * @param rowCount the number of rows to estimate for - * @return the estimated encoded size in bytes (always >= actual) - */ - public static long estimate(QwpTableBuffer tableBuffer, int rowCount) { - long size = 0; - - // Header: ILP4 (4) + version (1) + flags (1) + tableCount (2) + payloadLength (4) = 12 - size += HEADER_SIZE; - - // Table name - String tableName = tableBuffer.getTableName(); - int tableNameUtf8Len = NativeBufferWriter.utf8Length(tableName); - size += varintSize(tableNameUtf8Len) + tableNameUtf8Len; - - // Row count varint - size += varintSize(rowCount); - - int columnCount = tableBuffer.getColumnCount(); - // Column count varint - size += varintSize(columnCount); - - // Schema mode byte - size += 1; - - QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - - // Per-column schema: name + wire type code - for (int i = 0; i < columnCount; i++) { - QwpColumnDef colDef = columnDefs[i]; - int nameUtf8Len = NativeBufferWriter.utf8Length(colDef.getName()); - size += varintSize(nameUtf8Len) + nameUtf8Len; - size += 1; // wire type code - } - - // Per-column data - for (int i = 0; i < columnCount; i++) { - QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - QwpColumnDef colDef = columnDefs[i]; - int valueCount = col.getValueCount(); - - // Nullable bitmap - if (colDef.isNullable()) { - size += (rowCount + 7) / 8; - } - - // Safety margin: if this column has fewer values than rows, - // nextRow() will pad it -- account for one extra element - if (col.getSize() < rowCount) { - size += elementWireSize(col.getType()); - } - - size += estimateColumnData(col, valueCount); - } - - // Fixed safety margin - size += 8; - - return size; - } - - public static int varintSize(long value) { - if (value == 0) { - return 1; - } - return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; - } - - private static long estimateArrayColumn(QwpTableBuffer.ColumnBuffer col, int valueCount, int elemBytes) { - long size = 0; - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - if (dims == null || shapes == null) { - return size; - } - - int shapeIdx = 0; - for (int row = 0; row < valueCount; row++) { - int nDims = dims[row]; - // nDims byte - size += 1; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - // dim length int32 - size += 4; - elemCount *= shapes[shapeIdx++]; - } - // elements - size += (long) elemCount * elemBytes; - } - return size; - } - - private static long estimateColumnData(QwpTableBuffer.ColumnBuffer col, int valueCount) { - return switch (col.getType()) { - case TYPE_BOOLEAN -> (valueCount + 7) / 8; - case TYPE_BYTE -> valueCount; - case TYPE_SHORT, TYPE_CHAR -> (long) valueCount * 2; - case TYPE_INT, TYPE_FLOAT -> (long) valueCount * 4; - case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE -> (long) valueCount * 8; - case TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> - // Gorilla is disabled for UDP, so no encoding byte -- just raw longs - (long) valueCount * 8; - case TYPE_UUID -> (long) valueCount * 16; - case TYPE_LONG256 -> (long) valueCount * 32; - case TYPE_DECIMAL64 -> 1 + (long) valueCount * 8; - case TYPE_DECIMAL128 -> 1 + (long) valueCount * 16; - case TYPE_DECIMAL256 -> 1 + (long) valueCount * 32; - case TYPE_STRING, TYPE_VARCHAR -> - (long) (valueCount + 1) * 4 + col.getStringDataSize(); - case TYPE_SYMBOL -> estimateSymbolColumn(col, valueCount); - case TYPE_GEOHASH -> estimateGeoHashColumn(col, valueCount); - case TYPE_DOUBLE_ARRAY -> estimateArrayColumn(col, valueCount, 8); - case TYPE_LONG_ARRAY -> estimateArrayColumn(col, valueCount, 8); - default -> 0; - }; - } - - private static long estimateGeoHashColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) { - int precision = col.getGeoHashPrecision(); - if (precision < 1) { - precision = 1; - } - long size = varintSize(precision); - int valueSize = (precision + 7) / 8; - size += (long) valueCount * valueSize; - return size; - } - - private static long estimateSymbolColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) { - String[] dictionary = col.getSymbolDictionary(); - int dictSize = dictionary.length; - - long size = varintSize(dictSize); - for (String symbol : dictionary) { - int utf8Len = NativeBufferWriter.utf8Length(symbol); - size += varintSize(utf8Len) + utf8Len; - } - - // Per-value index varints. Maximum index is dictSize - 1. - int maxIndex = Math.max(0, dictSize - 1); - size += (long) valueCount * varintSize(maxIndex); - - return size; - } - - private static int elementWireSize(byte type) { - return switch (type) { - case TYPE_BOOLEAN -> 1; - case TYPE_BYTE -> 1; - case TYPE_SHORT, TYPE_CHAR -> 2; - case TYPE_INT, TYPE_FLOAT -> 4; - case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> 8; - case TYPE_UUID -> 16; - case TYPE_LONG256 -> 32; - case TYPE_DECIMAL64 -> 8; - case TYPE_DECIMAL128 -> 16; - case TYPE_DECIMAL256 -> 32; - case TYPE_STRING, TYPE_VARCHAR -> 4; // one offset entry - case TYPE_SYMBOL -> 1; // one varint index (at least 1 byte) - case TYPE_GEOHASH -> 8; - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> 5; // 1 dim byte + 4 shape int - default -> 0; - }; - } -} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 120af2a..c216f4b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -26,9 +26,11 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.cutlass.line.udp.UdpLineChannel; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; import io.questdb.client.std.Chars; @@ -73,8 +75,11 @@ public class QwpUdpSender implements Sender { private static final byte ENTRY_SYMBOL = 12; private static final byte ENTRY_TIMESTAMP_COL_MICROS = 13; private static final byte ENTRY_TIMESTAMP_COL_NANOS = 14; + private static final int VARINT_INT_UPPER_BOUND = 5; + private static final int SAFETY_MARGIN_BYTES = 8; private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); + private final ArraySizeCounter arraySizeCounter = new ArraySizeCounter(); private final NativeBufferWriter buffer = new NativeBufferWriter(); private final UdpLineChannel channel; private final QwpColumnWriter columnWriter = new QwpColumnWriter(); @@ -85,9 +90,14 @@ public class QwpUdpSender implements Sender { private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; private boolean closed; + private long committedEstimate; + private int committedEstimateColumnCount; + private int currentRowColumnCount; private QwpTableBuffer currentTableBuffer; private String currentTableName; + private int estimateColumnCount; private int rowJournalSize; + private long runningEstimate; public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { this(nf, interfaceIPv4, sendToAddress, port, ttl, 0); @@ -123,25 +133,14 @@ public void at(Instant timestamp) { public void atNow() { checkNotClosed(); checkTableSelected(); - if (maxDatagramSize > 0) { - maybeAutoFlush(); - } - currentTableBuffer.nextRow(); - rowJournalSize = 0; + commitCurrentRow(); } @Override public Sender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_BOOLEAN, false); - col.addBoolean(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_BOOL; - e.name = col.getName(); - e.boolValue = value; - } + appendBooleanColumn(columnName, value, true); return this; } @@ -155,6 +154,7 @@ public void cancelRow() { checkNotClosed(); if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); + rollbackEstimateToCommitted(); } rowJournalSize = 0; } @@ -163,6 +163,11 @@ public void cancelRow() { public void close() { if (!closed) { try { + if (hasInProgressRow()) { + currentTableBuffer.cancelCurrentRow(); + rollbackEstimateToCommitted(); + rowJournalSize = 0; + } flushInternal(); } catch (Exception e) { LOG.error("Error during close flush: {}", String.valueOf(e)); @@ -186,113 +191,78 @@ public void close() { @Override public Sender decimalColumn(CharSequence name, Decimal64 value) { - if (value == null || value.isNull()) return this; + if (value == null || value.isNull()) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL64, true); - col.addDecimal64(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DECIMAL64; - e.name = col.getName(); - e.objectValue = value; - } + appendDecimal64Column(name, value, true); return this; } @Override public Sender decimalColumn(CharSequence name, Decimal128 value) { - if (value == null || value.isNull()) return this; + if (value == null || value.isNull()) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL128, true); - col.addDecimal128(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DECIMAL128; - e.name = col.getName(); - e.objectValue = value; - } + appendDecimal128Column(name, value, true); return this; } @Override public Sender decimalColumn(CharSequence name, Decimal256 value) { - if (value == null || value.isNull()) return this; + if (value == null || value.isNull()) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL256, true); - col.addDecimal256(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DECIMAL256; - e.name = col.getName(); - e.objectValue = value; - } + appendDecimal256Column(name, value, true); return this; } @Override public Sender doubleArray(@NotNull CharSequence name, double[] values) { - if (values == null) return this; + if (values == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DOUBLE_ARRAY; - e.name = col.getName(); - e.objectValue = values; - } + appendDoubleArrayColumn(name, values, true); return this; } @Override public Sender doubleArray(@NotNull CharSequence name, double[][] values) { - if (values == null) return this; + if (values == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DOUBLE_ARRAY; - e.name = col.getName(); - e.objectValue = values; - } + appendDoubleArrayColumn(name, values, true); return this; } @Override public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { - if (values == null) return this; + if (values == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DOUBLE_ARRAY; - e.name = col.getName(); - e.objectValue = values; - } + appendDoubleArrayColumn(name, values, true); return this; } @Override public Sender doubleArray(CharSequence name, DoubleArray array) { - if (array == null) return this; + if (array == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(array); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DOUBLE_ARRAY; - e.name = col.getName(); - e.objectValue = array; - } + appendDoubleArrayColumn(name, array, true); return this; } @@ -300,84 +270,58 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public Sender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_DOUBLE, false); - col.addDouble(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DOUBLE; - e.name = col.getName(); - e.doubleValue = value; - } + appendDoubleColumn(columnName, value, true); return this; } @Override public void flush() { checkNotClosed(); + ensureNoInProgressRow("flush buffer"); flushInternal(); } @Override public Sender longArray(@NotNull CharSequence name, long[] values) { - if (values == null) return this; + if (values == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); - col.addLongArray(values); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_LONG_ARRAY; - e.name = col.getName(); - e.objectValue = values; - } + appendLongArrayColumn(name, values, true); return this; } @Override public Sender longArray(@NotNull CharSequence name, long[][] values) { - if (values == null) return this; + if (values == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); - col.addLongArray(values); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_LONG_ARRAY; - e.name = col.getName(); - e.objectValue = values; - } + appendLongArrayColumn(name, values, true); return this; } @Override public Sender longArray(@NotNull CharSequence name, long[][][] values) { - if (values == null) return this; + if (values == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); - col.addLongArray(values); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_LONG_ARRAY; - e.name = col.getName(); - e.objectValue = values; - } + appendLongArrayColumn(name, values, true); return this; } @Override public Sender longArray(@NotNull CharSequence name, LongArray array) { - if (array == null) return this; + if (array == null) { + return this; + } checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); - col.addLongArray(array); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_LONG_ARRAY; - e.name = col.getName(); - e.objectValue = array; - } + appendLongArrayColumn(name, array, true); return this; } @@ -385,14 +329,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public Sender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_LONG, false); - col.addLong(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_LONG; - e.name = col.getName(); - e.longValue = value; - } + appendLongColumn(columnName, value, true); return this; } @@ -411,20 +348,14 @@ public void reset() { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; rowJournalSize = 0; + resetEstimateState(); } @Override public Sender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_STRING, true); - col.addString(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_STRING; - e.name = col.getName(); - e.stringValue = Chars.toString(value); // todo: allocation! - } + appendStringColumn(columnName, value, true); return this; } @@ -432,14 +363,7 @@ public Sender stringColumn(CharSequence columnName, CharSequence value) { public Sender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_SYMBOL, true); - col.addSymbol(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_SYMBOL; - e.name = col.getName(); - e.stringValue = Chars.toString(value); // todo: allocation! - } + appendSymbolColumn(columnName, value, true); return this; } @@ -449,14 +373,16 @@ public Sender table(CharSequence tableName) { if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { return this; } - // Flush current table on switch if auto-flush is enabled and there are committed rows + ensureNoInProgressRow("switch tables"); if (maxDatagramSize > 0 && currentTableBuffer != null && currentTableBuffer.getRowCount() > 0) { flushSingleTable(currentTableName, currentTableBuffer); } cachedTimestampColumn = null; cachedTimestampNanosColumn = null; rowJournalSize = 0; - currentTableName = tableName.toString(); // todo: allocation! + resetEstimateState(); + + currentTableName = tableName.toString(); currentTableBuffer = tableBuffers.get(currentTableName); if (currentTableBuffer == null) { currentTableBuffer = new QwpTableBuffer(currentTableName); @@ -470,24 +396,10 @@ public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit un checkNotClosed(); checkTableSelected(); if (unit == ChronoUnit.NANOS) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP_NANOS, true); - col.addLong(value); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_TIMESTAMP_COL_NANOS; - e.name = col.getName(); - e.longValue = value; - } + appendTimestampColumn(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS, true); } else { long micros = toMicros(value, unit); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); - col.addLong(micros); - if (maxDatagramSize > 0) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_TIMESTAMP_COL_MICROS; - e.name = col.getName(); - e.longValue = micros; - } + appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS, true); } return this; } @@ -497,45 +409,335 @@ public Sender timestampColumn(CharSequence columnName, Instant value) { checkNotClosed(); checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); - col.addLong(micros); - if (maxDatagramSize > 0) { + appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS, true); + return this; + } + + private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean nullable) { + boolean exists = currentTableBuffer.hasColumn(name); + if (!exists && currentTableBuffer.getRowCount() > 0) { + if (currentRowColumnCount > 0) { + throw new LineSenderException("schema change in middle of row is not supported"); + } + flushSingleTable(currentTableName, currentTableBuffer); + } + + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, type, nullable); + syncSchemaEstimate(); + return col; + } + + private void appendBooleanColumn(CharSequence name, boolean value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addBoolean(value); + + long payloadDelta = packedBytes(col.getValueCount()) - packedBytes(valueCountBefore); + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_TIMESTAMP_COL_MICROS; + e.kind = ENTRY_BOOL; e.name = col.getName(); - e.longValue = micros; + e.boolValue = value; } - return this; } - private void atMicros(long timestampMicros) { - if (cachedTimestampColumn == null) { - cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + private void appendDecimal128Column(CharSequence name, Decimal128 value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addDecimal128(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 16; + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DECIMAL128; + e.name = col.getName(); + e.objectValue = value; } - cachedTimestampColumn.addLong(timestampMicros); - if (maxDatagramSize > 0) { + } + + private void appendDecimal256Column(CharSequence name, Decimal256 value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addDecimal256(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 32; + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_AT_MICROS; - e.longValue = timestampMicros; - maybeAutoFlush(); + e.kind = ENTRY_DECIMAL256; + e.name = col.getName(); + e.objectValue = value; } - currentTableBuffer.nextRow(); - rowJournalSize = 0; } - private void atNanos(long timestampNanos) { - if (cachedTimestampNanosColumn == null) { - cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); + private void appendDecimal64Column(CharSequence name, Decimal64 value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addDecimal64(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DECIMAL64; + e.name = col.getName(); + e.objectValue = value; } - cachedTimestampNanosColumn.addLong(timestampNanos); - if (maxDatagramSize > 0) { + } + + private void appendDesignatedTimestamp(long value, boolean nanos, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col; + if (nanos) { + if (cachedTimestampNanosColumn == null) { + cachedTimestampNanosColumn = acquireColumn("", TYPE_TIMESTAMP_NANOS, true); + } else { + syncSchemaEstimate(); + } + col = cachedTimestampNanosColumn; + } else { + if (cachedTimestampColumn == null) { + cachedTimestampColumn = acquireColumn("", TYPE_TIMESTAMP, true); + } else { + syncSchemaEstimate(); + } + col = cachedTimestampColumn; + } + + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addLong(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + + if (addJournal && maxDatagramSize > 0) { ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_AT_NANOS; - e.longValue = timestampNanos; - maybeAutoFlush(); + e.kind = nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS; + e.longValue = value; + } + } + + private void appendDoubleArrayColumn(CharSequence name, Object value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); + int sizeBefore = col.getSize(); + + long payloadDelta; + if (value instanceof double[] values) { + payloadDelta = estimateArrayValueSize(1, values.length); + col.addDoubleArray(values); + } else if (value instanceof double[][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + payloadDelta = estimateArrayValueSize(2, (long) dim0 * dim1); + col.addDoubleArray(values); + } else if (value instanceof double[][][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + payloadDelta = estimateArrayValueSize(3, (long) dim0 * dim1 * dim2); + col.addDoubleArray(values); + } else if (value instanceof DoubleArray values) { + payloadDelta = estimateArrayValueSize(values); + col.addDoubleArray(values); + } else { + throw new LineSenderException("unsupported double array type"); + } + + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DOUBLE_ARRAY; + e.name = col.getName(); + e.objectValue = value; + } + } + + private void appendDoubleColumn(CharSequence name, double value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addDouble(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_DOUBLE; + e.name = col.getName(); + e.doubleValue = value; + } + } + + private void appendLongArrayColumn(CharSequence name, Object value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); + int sizeBefore = col.getSize(); + + long payloadDelta; + if (value instanceof long[] values) { + payloadDelta = estimateArrayValueSize(1, values.length); + col.addLongArray(values); + } else if (value instanceof long[][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + payloadDelta = estimateArrayValueSize(2, (long) dim0 * dim1); + col.addLongArray(values); + } else if (value instanceof long[][][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + payloadDelta = estimateArrayValueSize(3, (long) dim0 * dim1 * dim2); + col.addLongArray(values); + } else if (value instanceof LongArray values) { + payloadDelta = estimateArrayValueSize(values); + col.addLongArray(values); + } else { + throw new LineSenderException("unsupported long array type"); + } + + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_LONG_ARRAY; + e.name = col.getName(); + e.objectValue = value; + } + } + + private void appendLongColumn(CharSequence name, long value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addLong(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_LONG; + e.name = col.getName(); + e.longValue = value; + } + } + + private void appendStringColumn(CharSequence name, CharSequence value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + long stringBytesBefore = col.getStringDataSize(); + col.addString(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 4 + + (col.getStringDataSize() - stringBytesBefore); + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_STRING; + e.name = col.getName(); + e.stringValue = Chars.toString(value); + } + } + + private void appendSymbolColumn(CharSequence name, CharSequence value, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + int dictSizeBefore = col.getSymbolDictionarySize(); + col.addSymbol(value); + + long payloadDelta = estimateSymbolPayloadDelta(col, valueCountBefore, dictSizeBefore, value); + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = ENTRY_SYMBOL; + e.name = col.getName(); + e.stringValue = Chars.toString(value); + } + } + + private void appendTimestampColumn(CharSequence name, byte type, long value, byte journalKind, boolean addJournal) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); + int sizeBefore = col.getSize(); + int valueCountBefore = col.getValueCount(); + col.addLong(value); + + long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; + applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); + currentRowColumnCount++; + + if (addJournal && maxDatagramSize > 0) { + ColumnEntry e = nextJournalEntry(); + e.kind = journalKind; + e.name = col.getName(); + e.longValue = value; + } + } + + private void applyRowPaddingEstimate(int targetRows) { + for (int i = 0, n = currentTableBuffer.getColumnCount(); i < n; i++) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); + int sizeBefore = col.getSize(); + int missing = targetRows - sizeBefore; + if (missing <= 0) { + continue; + } + + if (col.isNullable()) { + runningEstimate += bitmapBytes(sizeBefore + missing) - bitmapBytes(sizeBefore); + continue; + } + + int valuesBefore = col.getValueCount(); + runningEstimate += nonNullablePaddingCost(col.getType(), valuesBefore, missing); } - currentTableBuffer.nextRow(); - rowJournalSize = 0; + } + + private void applyValueEstimate(QwpTableBuffer.ColumnBuffer col, int sizeBefore, int sizeAfter, long payloadDelta) { + runningEstimate += payloadDelta; + if (col.isNullable()) { + runningEstimate += bitmapBytes(sizeAfter) - bitmapBytes(sizeBefore); + } + } + + private void atMicros(long timestampMicros) { + if (currentRowColumnCount == 0) { + throw new LineSenderException("no columns were provided"); + } + appendDesignatedTimestamp(timestampMicros, false, true); + commitCurrentRow(); + } + + private void atNanos(long timestampNanos) { + if (currentRowColumnCount == 0) { + throw new LineSenderException("no columns were provided"); + } + appendDesignatedTimestamp(timestampNanos, true, true); + commitCurrentRow(); } private void checkNotClosed() { @@ -550,9 +752,37 @@ private void checkTableSelected() { } } + private void commitCurrentRow() { + if (currentRowColumnCount == 0) { + throw new LineSenderException("no columns were provided"); + } + + int targetRows = currentTableBuffer.getRowCount() + 1; + applyRowPaddingEstimate(targetRows); + + if (maxDatagramSize > 0) { + maybeAutoFlush(); + } + + currentTableBuffer.nextRow(); + committedEstimate = runningEstimate; + committedEstimateColumnCount = estimateColumnCount; + currentRowColumnCount = 0; + rowJournalSize = 0; + } + + private void ensureNoInProgressRow(String operation) { + if (hasInProgressRow()) { + throw new LineSenderException( + "Cannot " + operation + " while row is in progress. " + + "Use sender.at(), sender.atNow(), or sender.cancelRow() first." + ); + } + } + private int encodeForUdp(QwpTableBuffer tableBuffer) { buffer.reset(); - // Write 12-byte ILP4 header: magic, version, flags=0, tableCount=1, payloadLength=0 (patched later) + // Write 12-byte QWP1 header: magic, version, flags=0, tableCount=1, payloadLength=0 (patched later) buffer.putByte((byte) 'Q'); buffer.putByte((byte) 'W'); buffer.putByte((byte) 'P'); @@ -569,13 +799,70 @@ private int encodeForUdp(QwpTableBuffer tableBuffer) { return buffer.getPosition(); } + private long estimateArrayValueSize(int nDims, long elementCount) { + return 1L + (long) nDims * 4 + elementCount * 8; + } + + private long estimateArrayValueSize(DoubleArray array) { + arraySizeCounter.reset(); + array.appendToBufPtr(arraySizeCounter); + return arraySizeCounter.size; + } + + private long estimateArrayValueSize(LongArray array) { + arraySizeCounter.reset(); + array.appendToBufPtr(arraySizeCounter); + return arraySizeCounter.size; + } + + private long estimateSymbolPayloadDelta( + QwpTableBuffer.ColumnBuffer col, + int valueCountBefore, + int dictSizeBefore, + CharSequence value + ) { + int valueCountAfter = col.getValueCount(); + if (valueCountAfter == valueCountBefore) { + return 0; + } + + int dictSizeAfter = col.getSymbolDictionarySize(); + if (dictSizeAfter == dictSizeBefore) { + int maxIndex = Math.max(0, dictSizeAfter - 1); + return NativeBufferWriter.varintSize(maxIndex); + } + + long delta = 0; + int utf8Len = utf8Length(value); + delta += NativeBufferWriter.varintSize(utf8Len) + utf8Len; + delta += NativeBufferWriter.varintSize(dictSizeAfter) + - NativeBufferWriter.varintSize(dictSizeBefore); + + if (dictSizeBefore > 0 && valueCountBefore > 0) { + int oldMax = dictSizeBefore - 1; + int newMax = dictSizeAfter - 1; + delta += (long) valueCountBefore * ( + NativeBufferWriter.varintSize(newMax) + - NativeBufferWriter.varintSize(oldMax) + ); + } + + int newMax = dictSizeAfter - 1; + delta += NativeBufferWriter.varintSize(newMax); + return delta; + } + private void flushInternal() { ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); - if (tableName == null) continue; + if (tableName == null) { + continue; + } QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) continue; + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } int len = encodeForUdp(tableBuffer); try { @@ -587,6 +874,8 @@ private void flushInternal() { } cachedTimestampColumn = null; cachedTimestampNanosColumn = null; + rowJournalSize = 0; + resetEstimateState(); } private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { @@ -599,33 +888,53 @@ private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { tableBuffer.reset(); cachedTimestampColumn = null; cachedTimestampNanosColumn = null; + resetEstimateState(); } private void maybeAutoFlush() { - int tentativeRowCount = currentTableBuffer.getRowCount() + 1; - long estimate = QwpDatagramSizeEstimator.estimate(currentTableBuffer, tentativeRowCount); - if (estimate > maxDatagramSize) { - if (currentTableBuffer.getRowCount() == 0) { - throw new LineSenderException( - "single row exceeds maximum datagram size (" + maxDatagramSize - + " bytes), estimated " + estimate + " bytes" - ); - } - currentTableBuffer.cancelCurrentRow(); - flushSingleTable(currentTableName, currentTableBuffer); - replayRowJournal(); - // Post-replay check: the replayed row alone may still exceed the limit. - tentativeRowCount = currentTableBuffer.getRowCount() + 1; - estimate = QwpDatagramSizeEstimator.estimate(currentTableBuffer, tentativeRowCount); - if (estimate > maxDatagramSize) { - throw new LineSenderException( - "single row exceeds maximum datagram size (" + maxDatagramSize - + " bytes), estimated " + estimate + " bytes" - ); - } + if (runningEstimate <= maxDatagramSize) { + return; + } + + if (currentTableBuffer.getRowCount() == 0) { + throw singleRowTooLarge(runningEstimate); + } + + currentTableBuffer.cancelCurrentRow(); + rollbackEstimateToCommitted(); + + flushSingleTable(currentTableName, currentTableBuffer); + replayRowJournal(); + applyRowPaddingEstimate(currentTableBuffer.getRowCount() + 1); + + if (runningEstimate > maxDatagramSize) { + throw singleRowTooLarge(runningEstimate); } } + private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) { + return switch (type) { + case TYPE_BOOLEAN -> packedBytes(valuesBefore + missing) - packedBytes(valuesBefore); + case TYPE_BYTE -> missing; + case TYPE_SHORT, TYPE_CHAR -> (long) missing * 2; + case TYPE_INT, TYPE_FLOAT -> (long) missing * 4; + case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> (long) missing * 8; + case TYPE_UUID -> (long) missing * 16; + case TYPE_LONG256 -> (long) missing * 32; + case TYPE_DECIMAL64 -> (long) missing * 8; + case TYPE_DECIMAL128 -> (long) missing * 16; + case TYPE_DECIMAL256 -> (long) missing * 32; + case TYPE_STRING, TYPE_VARCHAR -> (long) missing * 4; + case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> (long) missing * 5; + case TYPE_SYMBOL -> throw new IllegalStateException("symbol columns must be nullable"); + default -> 0; + }; + } + + private static int packedBytes(int valueCount) { + return (valueCount + 7) / 8; + } + private ColumnEntry nextJournalEntry() { if (rowJournalSize < rowJournal.size()) { ColumnEntry entry = rowJournal.getQuick(rowJournalSize); @@ -640,73 +949,92 @@ private ColumnEntry nextJournalEntry() { return entry; } - private void replayDoubleArray(ColumnEntry entry) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DOUBLE_ARRAY, true); - if (entry.objectValue instanceof double[] a) { - col.addDoubleArray(a); - } else if (entry.objectValue instanceof double[][] a) { - col.addDoubleArray(a); - } else if (entry.objectValue instanceof double[][][] a) { - col.addDoubleArray(a); - } else if (entry.objectValue instanceof DoubleArray a) { - col.addDoubleArray(a); - } - } - - private void replayLongArray(ColumnEntry entry) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(entry.name, TYPE_LONG_ARRAY, true); - if (entry.objectValue instanceof long[] a) { - col.addLongArray(a); - } else if (entry.objectValue instanceof long[][] a) { - col.addLongArray(a); - } else if (entry.objectValue instanceof long[][][] a) { - col.addLongArray(a); - } else if (entry.objectValue instanceof LongArray a) { - col.addLongArray(a); - } - } - private void replayRowJournal() { for (int i = 0; i < rowJournalSize; i++) { ColumnEntry entry = rowJournal.getQuick(i); switch (entry.kind) { - case ENTRY_AT_MICROS -> { - cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); - cachedTimestampColumn.addLong(entry.longValue); - } - case ENTRY_AT_NANOS -> { - cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); - cachedTimestampNanosColumn.addLong(entry.longValue); - } - case ENTRY_BOOL -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_BOOLEAN, false).addBoolean(entry.boolValue); - case ENTRY_DECIMAL128 -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DECIMAL128, true) - .addDecimal128((Decimal128) entry.objectValue); - case ENTRY_DECIMAL256 -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DECIMAL256, true) - .addDecimal256((Decimal256) entry.objectValue); - case ENTRY_DECIMAL64 -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DECIMAL64, true) - .addDecimal64((Decimal64) entry.objectValue); - case ENTRY_DOUBLE -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_DOUBLE, false).addDouble(entry.doubleValue); - case ENTRY_DOUBLE_ARRAY -> replayDoubleArray(entry); - case ENTRY_LONG -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_LONG, false).addLong(entry.longValue); - case ENTRY_LONG_ARRAY -> replayLongArray(entry); - case ENTRY_STRING -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_STRING, true).addString(entry.stringValue); - case ENTRY_SYMBOL -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_SYMBOL, true).addSymbol(entry.stringValue); + case ENTRY_AT_MICROS -> appendDesignatedTimestamp(entry.longValue, false, false); + case ENTRY_AT_NANOS -> appendDesignatedTimestamp(entry.longValue, true, false); + case ENTRY_BOOL -> appendBooleanColumn(entry.name, entry.boolValue, false); + case ENTRY_DECIMAL128 -> appendDecimal128Column(entry.name, (Decimal128) entry.objectValue, false); + case ENTRY_DECIMAL256 -> appendDecimal256Column(entry.name, (Decimal256) entry.objectValue, false); + case ENTRY_DECIMAL64 -> appendDecimal64Column(entry.name, (Decimal64) entry.objectValue, false); + case ENTRY_DOUBLE -> appendDoubleColumn(entry.name, entry.doubleValue, false); + case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayColumn(entry.name, entry.objectValue, false); + case ENTRY_LONG -> appendLongColumn(entry.name, entry.longValue, false); + case ENTRY_LONG_ARRAY -> appendLongArrayColumn(entry.name, entry.objectValue, false); + case ENTRY_STRING -> appendStringColumn(entry.name, entry.stringValue, false); + case ENTRY_SYMBOL -> appendSymbolColumn(entry.name, entry.stringValue, false); case ENTRY_TIMESTAMP_COL_MICROS -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_TIMESTAMP, true).addLong(entry.longValue); + appendTimestampColumn(entry.name, TYPE_TIMESTAMP, entry.longValue, ENTRY_TIMESTAMP_COL_MICROS, false); case ENTRY_TIMESTAMP_COL_NANOS -> - currentTableBuffer.getOrCreateColumn(entry.name, TYPE_TIMESTAMP_NANOS, true).addLong(entry.longValue); + appendTimestampColumn(entry.name, TYPE_TIMESTAMP_NANOS, entry.longValue, ENTRY_TIMESTAMP_COL_NANOS, false); + default -> throw new LineSenderException("unknown row journal entry type: " + entry.kind); } } } + private void resetEstimateState() { + runningEstimate = 0; + committedEstimate = 0; + estimateColumnCount = 0; + committedEstimateColumnCount = 0; + currentRowColumnCount = 0; + } + + private boolean hasInProgressRow() { + return currentTableBuffer != null && currentTableBuffer.hasInProgressRow(); + } + + private void rollbackEstimateToCommitted() { + runningEstimate = committedEstimate; + estimateColumnCount = committedEstimateColumnCount; + currentRowColumnCount = 0; + } + + private LineSenderException singleRowTooLarge(long estimate) { + return new LineSenderException( + "single row exceeds maximum datagram size (" + maxDatagramSize + + " bytes), estimated " + estimate + " bytes" + ); + } + + private void syncSchemaEstimate() { + int newColumnCount = currentTableBuffer.getColumnCount(); + if (newColumnCount == estimateColumnCount) { + return; + } + + if (estimateColumnCount == 0) { + long base = HEADER_SIZE; + int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName); + base += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; + base += VARINT_INT_UPPER_BOUND; // row count varint upper bound + base += VARINT_INT_UPPER_BOUND; // column count varint upper bound + base += 1; // schema mode byte + base += SAFETY_MARGIN_BYTES; + runningEstimate += base; + } + + QwpColumnDef[] defs = currentTableBuffer.getColumnDefs(); + for (int i = estimateColumnCount; i < newColumnCount; i++) { + QwpColumnDef def = defs[i]; + int nameUtf8 = NativeBufferWriter.utf8Length(def.getName()); + runningEstimate += NativeBufferWriter.varintSize(nameUtf8) + nameUtf8; + runningEstimate += 1; // wire type code + + byte type = def.getTypeCode(); + if (type == TYPE_STRING || type == TYPE_VARCHAR) { + runningEstimate += 4; // offset[0] + } else if (type == TYPE_SYMBOL) { + runningEstimate += 1; // varintSize(0) for empty dictionary length + } else if (type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128 || type == TYPE_DECIMAL256) { + runningEstimate += 1; // scale byte + } + } + estimateColumnCount = newColumnCount; + } + private long toMicros(long value, ChronoUnit unit) { return switch (unit) { case NANOS -> value / 1000L; @@ -720,6 +1048,66 @@ private long toMicros(long value, ChronoUnit unit) { }; } + private static int bitmapBytes(int size) { + return (size + 7) / 8; + } + + private static int utf8Length(CharSequence s) { + if (s == null) { + return 0; + } + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { + i++; + len += 4; + } else if (Character.isSurrogate(c)) { + len++; + } else { + len += 3; + } + } + return len; + } + + private static final class ArraySizeCounter implements ArrayBufferAppender { + private long size; + + @Override + public void putBlockOfBytes(long from, long len) { + size += len; + } + + @Override + public void putByte(byte b) { + size++; + } + + @Override + public void putDouble(double value) { + size += 8; + } + + @Override + public void putInt(int value) { + size += 4; + } + + @Override + public void putLong(long value) { + size += 8; + } + + private void reset() { + size = 0; + } + } + private static class ColumnEntry { boolean boolValue; double doubleValue; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 5cc8959..8676021 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -149,6 +149,10 @@ public int getColumnCount() { return columns.size(); } + public boolean hasColumn(CharSequence name) { + return columnNameToIndex.get(name) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + } + /** * Returns the column definitions (cached for efficiency). */ @@ -227,6 +231,15 @@ public int getRowCount() { return rowCount; } + public boolean hasInProgressRow() { + for (int i = 0, n = columns.size(); i < n; i++) { + if (fastColumns[i].size > rowCount) { + return true; + } + } + return false; + } + /** * Returns the schema hash for this table. *

    @@ -1072,10 +1085,18 @@ public String[] getSymbolDictionary() { return dict; } + public int getSymbolDictionarySize() { + return symbolList != null ? symbolList.size() : 0; + } + public byte getType() { return type; } + public boolean isNullable() { + return nullable; + } + public int getValueCount() { return valueCount; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 1214b45..1f806a9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -497,6 +497,20 @@ public void testWriteVarintLarge() throws Exception { }); } + @Test + public void testVarintSizeMatchesEncodedLength() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + long[] values = {0, 1, 127, 128, 16383, 16384, 2_097_151, 2_097_152, Long.MAX_VALUE}; + for (long value : values) { + writer.reset(); + writer.putVarint(value); + Assert.assertEquals("value=" + value, NativeBufferWriter.varintSize(value), writer.getPosition()); + } + } + }); + } + @Test public void testWriteVarintMedium() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java deleted file mode 100644 index fd080ba..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDatagramSizeEstimatorTest.java +++ /dev/null @@ -1,564 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client; - -import io.questdb.client.cutlass.qwp.client.QwpDatagramSizeEstimator; -import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; -import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.Decimal128; -import io.questdb.client.std.Decimal256; -import io.questdb.client.std.Decimal64; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Random; - -import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; - -public class QwpDatagramSizeEstimatorTest { - - @Test - public void testBooleanColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_BOOLEAN, false).addBoolean(true); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testByteColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_BYTE, false).addByte((byte) 42); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testCharColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_CHAR, false).addShort((short) 'A'); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDateColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DATE, false).addLong(1_700_000_000_000L); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDecimal128Column() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DECIMAL128, true).addDecimal128(new Decimal128(0, 12345, 2)); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDecimal256Column() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DECIMAL256, true).addDecimal256(new Decimal256(0, 0, 0, 12345, 2)); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDecimal64Column() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DECIMAL64, true).addDecimal64(new Decimal64(12345, 2)); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDoubleArray2DColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DOUBLE_ARRAY, true).addDoubleArray( - new double[][]{{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}} - ); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDoubleArray3DColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DOUBLE_ARRAY, true).addDoubleArray( - new double[][][]{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}} - ); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDoubleArrayColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DOUBLE_ARRAY, true).addDoubleArray(new double[]{1.0, 2.0, 3.0}); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testDoubleColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_DOUBLE, false).addDouble(3.14); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testFloatColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_FLOAT, false).addFloat(3.14f); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testGeoHashColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_GEOHASH, true).addGeoHash(0x1234L, 20); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testIntColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_INT, false).addInt(42); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testLong256Column() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_LONG256, false).addLong256(1L, 2L, 3L, 4L); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testLongArray2DColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_LONG_ARRAY, true).addLongArray( - new long[][]{{10L, 20L, 30L}, {40L, 50L, 60L}} - ); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testLongArrayColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_LONG_ARRAY, true).addLongArray(new long[]{10L, 20L, 30L}); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testLongColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_LONG, false).addLong(42L); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testMultiByteUtf8() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("\u6e2c\u5b9a")) { // 測定 (3 bytes per char) - buf.getOrCreateColumn("\u6e29\u5ea6", TYPE_DOUBLE, false).addDouble(22.5); // 温度 - buf.getOrCreateColumn("\u30e1\u30e2", TYPE_STRING, true).addString("\u3053\u3093\u306b\u3061\u306f"); // メモ, こんにちは - buf.getOrCreateColumn("\u5730\u57df", TYPE_SYMBOL, true).addSymbol("\u6771\u4eac"); // 地域, 東京 - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testMultiRowDoubleTimestamp() throws Exception { - assertMemoryLeak(() -> { - for (int rowCount : new int[]{1, 5, 10, 50}) { - try (QwpTableBuffer buf = new QwpTableBuffer("measurements")) { - for (int i = 0; i < rowCount; i++) { - buf.getOrCreateColumn("value", TYPE_DOUBLE, false).addDouble(i * 1.1); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); - buf.nextRow(); - } - assertEstimateAccuracy(buf, rowCount); - } - } - }); - } - - @Test - public void testNullableColumnMixedNullNonNull() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - QwpTableBuffer.ColumnBuffer idCol = buf.getOrCreateColumn("id", TYPE_LONG, false); - QwpTableBuffer.ColumnBuffer valCol = buf.getOrCreateColumn("val", TYPE_DOUBLE, true); - QwpTableBuffer.ColumnBuffer tsCol = buf.getOrCreateColumn("", TYPE_TIMESTAMP, true); - - // Row 1: has value - idCol.addLong(1L); - valCol.addDouble(10.0); - tsCol.addLong(1_000_000L); - buf.nextRow(); - - // Row 2: null (skip val) - idCol.addLong(2L); - tsCol.addLong(2_000_000L); - buf.nextRow(); - - // Row 3: has value - idCol.addLong(3L); - valCol.addDouble(30.0); - tsCol.addLong(3_000_000L); - buf.nextRow(); - - // Row 4: null - idCol.addLong(4L); - tsCol.addLong(4_000_000L); - buf.nextRow(); - - // Row 5: has value - idCol.addLong(5L); - valCol.addDouble(50.0); - tsCol.addLong(5_000_000L); - buf.nextRow(); - - assertEstimateAccuracy(buf, 5); - } - }); - } - - @Test - public void testShortColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_SHORT, false).addShort((short) 42); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testStringColumnEmpty() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_STRING, true).addString(""); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testStringColumnLong() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_STRING, true).addString("a]".repeat(500)); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testStringColumnShort() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_STRING, true).addString("hello"); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testSymbolWith100DistinctValues() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - for (int i = 0; i < 100; i++) { - buf.getOrCreateColumn("sym", TYPE_SYMBOL, true).addSymbol("val-" + i); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); - buf.nextRow(); - } - assertEstimateAccuracy(buf, 100); - } - }); - } - - @Test - public void testSymbolWith10DistinctValues() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - for (int i = 0; i < 10; i++) { - buf.getOrCreateColumn("sym", TYPE_SYMBOL, true).addSymbol("value-" + i); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); - buf.nextRow(); - } - assertEstimateAccuracy(buf, 10); - } - }); - } - - @Test - public void testSymbolWith1DistinctValue() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - for (int i = 0; i < 5; i++) { - buf.getOrCreateColumn("sym", TYPE_SYMBOL, true).addSymbol("only-one"); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + i); - buf.nextRow(); - } - assertEstimateAccuracy(buf, 5); - } - }); - } - - @Test - public void testTimestampColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_TIMESTAMP, true).addLong(1_700_000_000L); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testTimestampNanosColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_TIMESTAMP_NANOS, true).addLong(1_700_000_000_000L); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testUuidColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_UUID, false).addUuid(123L, 456L); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testRandomSchemas() throws Exception { - assertMemoryLeak(() -> { - Random rng = new Random(42); - byte[] fixedTypes = { - TYPE_BOOLEAN, TYPE_BYTE, TYPE_SHORT, TYPE_INT, TYPE_LONG, - TYPE_FLOAT, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, - TYPE_UUID, TYPE_LONG256 - }; - - for (int trial = 0; trial < 100; trial++) { - int numCols = 1 + rng.nextInt(5); - int numRows = 1 + rng.nextInt(20); - - try (QwpTableBuffer buf = new QwpTableBuffer("random_" + trial)) { - byte[] colTypes = new byte[numCols]; - for (int c = 0; c < numCols; c++) { - int pick = rng.nextInt(fixedTypes.length + 2); - if (pick < fixedTypes.length) { - colTypes[c] = fixedTypes[pick]; - } else if (pick == fixedTypes.length) { - colTypes[c] = TYPE_STRING; - } else { - colTypes[c] = TYPE_SYMBOL; - } - } - - for (int row = 0; row < numRows; row++) { - for (int c = 0; c < numCols; c++) { - boolean isNullable = colTypes[c] == TYPE_STRING || colTypes[c] == TYPE_SYMBOL - || colTypes[c] == TYPE_TIMESTAMP; - QwpTableBuffer.ColumnBuffer col = buf.getOrCreateColumn( - "c" + c, colTypes[c], isNullable - ); - addRandomValue(col, colTypes[c], rng); - } - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L + row); - buf.nextRow(); - } - assertEstimateAccuracy(buf, numRows); - } - } - }); - } - - @Test - public void testVarcharColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer buf = new QwpTableBuffer("t")) { - buf.getOrCreateColumn("v", TYPE_VARCHAR, true).addString("varchar value"); - buf.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1_000_000L); - buf.nextRow(); - assertEstimateAccuracy(buf, 1); - } - }); - } - - @Test - public void testVarintSize() { - Assert.assertEquals(1, QwpDatagramSizeEstimator.varintSize(0)); - Assert.assertEquals(1, QwpDatagramSizeEstimator.varintSize(1)); - Assert.assertEquals(1, QwpDatagramSizeEstimator.varintSize(127)); - Assert.assertEquals(2, QwpDatagramSizeEstimator.varintSize(128)); - Assert.assertEquals(2, QwpDatagramSizeEstimator.varintSize(16383)); - Assert.assertEquals(3, QwpDatagramSizeEstimator.varintSize(16384)); - } - - private static void addRandomValue(QwpTableBuffer.ColumnBuffer col, byte type, Random rng) { - switch (type) { - case TYPE_BOOLEAN -> col.addBoolean(rng.nextBoolean()); - case TYPE_BYTE -> col.addByte((byte) rng.nextInt()); - case TYPE_SHORT -> col.addShort((short) rng.nextInt()); - case TYPE_INT -> col.addInt(rng.nextInt()); - case TYPE_LONG -> col.addLong(rng.nextLong()); - case TYPE_FLOAT -> col.addFloat(rng.nextFloat()); - case TYPE_DOUBLE -> col.addDouble(rng.nextDouble()); - case TYPE_DATE -> col.addLong(rng.nextLong()); - case TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> col.addLong(Math.abs(rng.nextLong())); - case TYPE_UUID -> col.addUuid(rng.nextLong(), rng.nextLong()); - case TYPE_LONG256 -> col.addLong256(rng.nextLong(), rng.nextLong(), rng.nextLong(), rng.nextLong()); - case TYPE_STRING -> col.addString("str" + rng.nextInt(1000)); - case TYPE_SYMBOL -> col.addSymbol("sym" + rng.nextInt(20)); - } - } - - private static void assertEstimateAccuracy(QwpTableBuffer buf, int rowCount) { - long estimate = QwpDatagramSizeEstimator.estimate(buf, rowCount); - - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { - encoder.setGorillaEnabled(false); - int actual = encoder.encode(buf, false); - - Assert.assertTrue( - "estimate (" + estimate + ") < actual (" + actual + ")", - estimate >= actual - ); - Assert.assertTrue( - "estimate (" + estimate + ") - actual (" + actual + ") = " + (estimate - actual) + " >= 32", - estimate - actual < 32 - ); - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java new file mode 100644 index 0000000..4562dea --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -0,0 +1,1254 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpUdpSender; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.network.NetworkFacadeImpl; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.SCHEMA_MODE_FULL; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_BOOLEAN; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DECIMAL128; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DECIMAL256; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DECIMAL64; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DOUBLE; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DOUBLE_ARRAY; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_LONG; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_LONG_ARRAY; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_STRING; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_SYMBOL; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_TIMESTAMP; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_TIMESTAMP_NANOS; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_VARCHAR; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.VERSION_1; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpUdpSenderTest { + + @Test + public void testFirstRowAllowsMultipleNewColumnsAndEncodesRow() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("a", 1) + .doubleColumn("b", 2.0) + .stringColumn("c", "x") + .atNow(), + "a", 1L, + "b", 2.0, + "c", "x") + ); + + RunResult result = runScenario(rows, 1024 * 1024); + Assert.assertEquals(1, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderMixedNullablePaddingPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String omega = repeat('z', 256); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .symbol("sym", "v1") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(), + "sym", "v1", + "x", 1L, + "s", alpha), + row("t", sender -> sender.table("t") + .symbol("sym", null) + .longColumn("x", 2) + .atNow(), + "sym", null, + "x", 2L, + "s", null), + row("t", sender -> sender.table("t") + .symbol("sym", null) + .longColumn("x", 3) + .stringColumn("s", null) + .atNow(), + "sym", null, + "x", 3L, + "s", null), + row("t", sender -> sender.table("t") + .symbol("sym", "v2") + .longColumn("x", 4) + .stringColumn("s", omega) + .atNow(), + "sym", "v2", + "x", 4L, + "s", omega) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderArrayReplayPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + long[] longValues = new long[128]; + double[] doubleValues = new double[128]; + for (int i = 0; i < 128; i++) { + longValues[i] = i * 3L; + doubleValues[i] = i * 1.25; + } + + long[][] longMatrix = new long[8][8]; + double[][] doubleMatrix = new double[8][8]; + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + longMatrix[i][j] = i * 100L + j; + doubleMatrix[i][j] = i * 10.0 + j + 0.5; + } + } + + List rows = Arrays.asList( + row("arrays", sender -> sender.table("arrays") + .symbol("sym", "alpha") + .longColumn("x", 1) + .longArray("la", longValues) + .doubleArray("da", doubleValues) + .atNow(), + "sym", "alpha", + "x", 1L, + "la", longArrayValue(shape(128), longValues), + "da", doubleArrayValue(shape(128), doubleValues)), + row("arrays", sender -> sender.table("arrays") + .symbol("sym", "beta") + .longColumn("x", 2) + .longArray("la", longMatrix) + .doubleArray("da", doubleMatrix) + .atNow(), + "sym", "beta", + "x", 2L, + "la", longArrayValue(shape(8, 8), flatten(longMatrix)), + "da", doubleArrayValue(shape(8, 8), flatten(doubleMatrix))), + row("arrays", sender -> sender.table("arrays") + .symbol("sym", "gamma") + .longColumn("x", 3) + .longArray("la", (long[]) null) + .doubleArray("da", (double[]) null) + .atNow(), + "sym", "gamma", + "x", 3L, + "la", null, + "da", null) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderMixedTypesPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String msg1 = repeat('m', 240); + String msg2 = repeat('n', 240); + List rows = Arrays.asList( + row("mix", sender -> sender.table("mix") + .symbol("sym", "alpha") + .boolColumn("active", true) + .longColumn("count", 1) + .doubleColumn("value", 1.5) + .stringColumn("msg", msg1) + .decimalColumn("d64", Decimal64.fromLong(12345L, 2)) + .decimalColumn("d128", Decimal128.fromLong(678901234L, 4)) + .decimalColumn("d256", Decimal256.fromLong(9876543210L, 3)) + .timestampColumn("eventMicros", 123456L, ChronoUnit.MICROS) + .timestampColumn("eventNanos", 999L, ChronoUnit.NANOS) + .at(1_000_000L, ChronoUnit.MICROS), + "sym", "alpha", + "active", true, + "count", 1L, + "value", 1.5, + "msg", msg1, + "d64", decimal(12345L, 2), + "d128", decimal(678901234L, 4), + "d256", decimal(9876543210L, 3), + "eventMicros", 123456L, + "eventNanos", 999L, + "", 1_000_000L), + row("mix", sender -> sender.table("mix") + .symbol("sym", "beta") + .boolColumn("active", false) + .longColumn("count", 2) + .doubleColumn("value", 2.5) + .stringColumn("msg", msg2) + .decimalColumn("d64", Decimal64.fromLong(-67890L, 2)) + .decimalColumn("d128", Decimal128.fromLong(2222333344L, 4)) + .decimalColumn("d256", Decimal256.fromLong(7777777770L, 3)) + .timestampColumn("eventMicros", 654321L, ChronoUnit.MICROS) + .timestampColumn("eventNanos", 12345L, ChronoUnit.NANOS) + .at(2_000_000L, ChronoUnit.MICROS), + "sym", "beta", + "active", false, + "count", 2L, + "value", 2.5, + "msg", msg2, + "d64", decimal(-67890L, 2), + "d128", decimal(2222333344L, 4), + "d256", decimal(7777777770L, 3), + "eventMicros", 654321L, + "eventNanos", 12345L, + "", 2_000_000L) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testMixingAtNowAndAtMicrosAfterCommittedRowsThrowsSchemaChange() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("x", 1) + .atNow(); + + sender.longColumn("x", 2); + assertThrowsContains("schema change in middle of row is not supported", + () -> sender.at(2, ChronoUnit.MICROS)); + sender.cancelRow(); + } + }); + } + + @Test + public void testFlushWhileRowInProgressThrowsAndPreservesRow() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t").longColumn("x", 1); + + assertThrowsContains("Cannot flush buffer while row is in progress", sender::flush); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(1, nf.sendCount); + assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testSwitchTableWhileRowInProgressThrowsAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t1").longColumn("x", 1); + + assertThrowsContains("Cannot switch tables while row is in progress", + () -> sender.table("t2")); + + sender.atNow(); + sender.table("t2").longColumn("y", 2).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqualIgnoringOrder( + Arrays.asList(decodedRow("t1", "x", 1L), decodedRow("t2", "y", 2L)), + decodeRows(nf.packets) + ); + }); + } + + @Test + public void testCloseDropsInProgressRowButFlushesCommittedRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t").longColumn("x", 1).atNow(); + sender.longColumn("x", 2); + } + + Assert.assertEquals(1, nf.sendCount); + assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testOversizedSingleRowRejectedAfterReplayUsesActualEncodedSize() throws Exception { + assertMemoryLeak(() -> { + String small = repeat('s', 32); + String large = repeat('x', 5000); + List largeRow = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 2) + .stringColumn("s", large) + .atNow(), + "x", 2L, + "s", large) + ); + int maxDatagramSize = fullPacketSize(largeRow) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("x", 1) + .stringColumn("s", small) + .atNow(); + + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("x", 2) + .stringColumn("s", large) + .atNow() + ); + } + + Assert.assertEquals(1, nf.sendCount); + assertPacketsWithinLimit(new RunResult(nf.packets, nf.lengths, nf.sendCount), maxDatagramSize); + assertRowsEqual( + Arrays.asList(decodedRow("t", "x", 1L, "s", small)), + decodeRows(nf.packets) + ); + }); + } + + @Test + public void testOversizedSingleRowRejectedBeforeReplayUsesActualEncodedSize() throws Exception { + assertMemoryLeak(() -> { + String large = repeat('x', 5000); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 1) + .stringColumn("s", large) + .atNow(), + "x", 1L, + "s", large) + ); + int maxDatagramSize = fullPacketSize(rows) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t"); + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("x", 1) + .stringColumn("s", large) + .atNow() + ); + } + + Assert.assertEquals(0, nf.sendCount); + }); + } + + @Test + public void testOversizedArrayRowRejectedUsesActualEncodedSize() throws Exception { + assertMemoryLeak(() -> { + long[] longValues = new long[1024]; + double[] doubleValues = new double[1024]; + for (int i = 0; i < 1024; i++) { + longValues[i] = i; + doubleValues[i] = i + 0.25; + } + + List rows = Arrays.asList( + row("arrays", sender -> sender.table("arrays") + .longColumn("x", 1) + .longArray("la", longValues) + .doubleArray("da", doubleValues) + .atNow(), + "x", 1L, + "la", longArrayValue(shape(1024), longValues), + "da", doubleArrayValue(shape(1024), doubleValues)) + ); + int maxDatagramSize = fullPacketSize(rows) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("arrays"); + assertThrowsContains("single row exceeds maximum datagram size", () -> + sender.longColumn("x", 1) + .longArray("la", longValues) + .doubleArray("da", doubleValues) + .atNow() + ); + } + + Assert.assertEquals(0, nf.sendCount); + }); + } + + @Test + public void testSchemaChangeMidRowThrows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + assertThrowsContains("schema change in middle of row is not supported", () -> sender.longColumn("b", 3)); + + sender.cancelRow(); + Assert.assertEquals(0, nf.sendCount); + } + + Assert.assertEquals(1, nf.sendCount); + assertRowsEqual(Arrays.asList(decodedRow("t", "a", 1L)), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeWithCommittedRowsFlushesImmediately() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + Assert.assertEquals(0, nf.sendCount); + + sender.longColumn("b", 2); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + Assert.assertEquals(1, nf.sendCount); + + sender.flush(); + Assert.assertEquals(2, nf.sendCount); + } + }); + } + + @Test + public void testSymbolBoundary16383To16384PreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + List rows = new ArrayList<>(16_390); + for (int i = 0; i < 16_390; i++) { + final int value = i; + rows.add(row("t", sender -> sender.table("t") + .symbol("sym", "v" + value) + .longColumn("x", value) + .atNow(), + "sym", "v" + value, + "x", (long) value)); + } + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testSymbolBoundary127To128PreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + List rows = new ArrayList<>(130); + for (int i = 0; i < 130; i++) { + final int value = i; + rows.add(row("t", sender -> sender.table("t") + .symbol("sym", "v" + value) + .longColumn("x", value) + .atNow(), + "sym", "v" + value, + "x", (long) value)); + } + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testZeroColumnRowsThrow() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t"); + + assertThrowsContains("no columns were provided", sender::atNow); + assertThrowsContains("no columns were provided", () -> sender.at(1, ChronoUnit.MICROS)); + assertThrowsContains("no columns were provided", () -> sender.at(1, ChronoUnit.NANOS)); + } + }); + } + + private static void assertPacketsWithinLimit(RunResult result, int maxDatagramSize) { + for (int i = 0; i < result.lengths.size(); i++) { + int len = result.lengths.get(i); + Assert.assertTrue( + "packet " + i + " exceeds maxDatagramSize: " + len + " > " + maxDatagramSize, + len <= maxDatagramSize + ); + } + } + + private static void assertRowsEqual(List expected, List actual) { + Assert.assertEquals("row count mismatch", expected.size(), actual.size()); + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals("row mismatch at index " + i, expected.get(i), actual.get(i)); + } + } + + private static void assertRowsEqualIgnoringOrder(List expected, List actual) { + Map counts = new HashMap<>(); + for (DecodedRow row : expected) { + counts.merge(row, 1, Integer::sum); + } + for (DecodedRow row : actual) { + Integer count = counts.get(row); + if (count == null) { + Assert.fail("unexpected row: " + row); + } + if (count == 1) { + counts.remove(row); + } else { + counts.put(row, count - 1); + } + } + if (!counts.isEmpty()) { + Assert.fail("missing rows: " + counts); + } + } + + private static void assertThrowsContains(String expected, ThrowingRunnable runnable) { + try { + runnable.run(); + Assert.fail("Expected exception containing: " + expected); + } catch (LineSenderException e) { + Assert.assertTrue("Expected message to contain '" + expected + "' but got: " + e.getMessage(), + e.getMessage().contains(expected)); + } catch (Exception e) { + throw new AssertionError("Unexpected exception type", e); + } + } + + private static BigDecimal decimal(long unscaled, int scale) { + return BigDecimal.valueOf(unscaled, scale); + } + + private static DecodedRow decodedRow(String table, Object... kvs) { + return new DecodedRow(table, columns(kvs)); + } + + private static List decodeRows(List packets) { + ArrayList rows = new ArrayList<>(); + for (byte[] packet : packets) { + rows.addAll(new DatagramDecoder(packet).decode()); + } + return rows; + } + + private static DoubleArrayValue doubleArrayValue(int[] shape, double... values) { + ArrayList dims = new ArrayList<>(shape.length); + for (int dim : shape) { + dims.add(dim); + } + ArrayList elems = new ArrayList<>(values.length); + for (double value : values) { + elems.add(value); + } + return new DoubleArrayValue(dims, elems); + } + + private static List expectedRows(List rows) { + ArrayList expected = new ArrayList<>(rows.size()); + for (ScenarioRow row : rows) { + expected.add(row.expected); + } + return expected; + } + + private static double[] flatten(double[][] matrix) { + int size = 0; + for (double[] row : matrix) { + size += row.length; + } + double[] flat = new double[size]; + int offset = 0; + for (double[] row : matrix) { + System.arraycopy(row, 0, flat, offset, row.length); + offset += row.length; + } + return flat; + } + + private static int fullPacketSize(List rows) throws Exception { + RunResult result = runScenario(rows, 0); + Assert.assertEquals("expected a single unbounded packet", 1, result.packets.size()); + return result.lengths.get(0); + } + + private static long[] flatten(long[][] matrix) { + int size = 0; + for (long[] row : matrix) { + size += row.length; + } + long[] flat = new long[size]; + int offset = 0; + for (long[] row : matrix) { + System.arraycopy(row, 0, flat, offset, row.length); + offset += row.length; + } + return flat; + } + + private static LinkedHashMap columns(Object... kvs) { + if ((kvs.length & 1) != 0) { + throw new IllegalArgumentException("key/value pairs expected"); + } + LinkedHashMap columns = new LinkedHashMap<>(); + for (int i = 0; i < kvs.length; i += 2) { + columns.put((String) kvs[i], kvs[i + 1]); + } + return columns; + } + + private static LongArrayValue longArrayValue(int[] shape, long... values) { + ArrayList dims = new ArrayList<>(shape.length); + for (int dim : shape) { + dims.add(dim); + } + ArrayList elems = new ArrayList<>(values.length); + for (long value : values) { + elems.add(value); + } + return new LongArrayValue(dims, elems); + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, value); + return new String(chars); + } + + private static ScenarioRow row(String table, ThrowingConsumer writer, Object... kvs) { + return new ScenarioRow(decodedRow(table, kvs), writer); + } + + private static RunResult runScenario(List rows, int maxDatagramSize) throws Exception { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + if (maxDatagramSize > 0) { + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + for (ScenarioRow row : rows) { + row.writer.accept(sender); + } + sender.flush(); + } + } else { + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + for (ScenarioRow row : rows) { + row.writer.accept(sender); + } + sender.flush(); + } + } + return new RunResult(nf.packets, nf.lengths, nf.sendCount); + } + + private static int[] shape(int... dims) { + return dims; + } + + @FunctionalInterface + private interface ThrowingConsumer { + void accept(T value) throws Exception; + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + + private static final class CapturingNetworkFacade extends NetworkFacadeImpl { + private final List lengths = new ArrayList<>(); + private final List packets = new ArrayList<>(); + private int sendCount; + + @Override + public int close(int fd) { + return 0; + } + + @Override + public void freeSockAddr(long pSockaddr) { + // no-op + } + + @Override + public int sendToRaw(int fd, long lo, int len, long socketAddress) { + byte[] packet = new byte[len]; + Unsafe.getUnsafe().copyMemory(null, lo, packet, Unsafe.BYTE_OFFSET, len); + packets.add(packet); + lengths.add(len); + sendCount++; + return len; + } + + @Override + public int setMulticastInterface(int fd, int ipv4Address) { + return 0; + } + + @Override + public int setMulticastTtl(int fd, int ttl) { + return 0; + } + + @Override + public long sockaddr(int address, int port) { + return 1; + } + + @Override + public int socketUdp() { + return 1; + } + } + + private static final class ColumnValues { + private final Object[] rows; + + private ColumnValues(Object[] rows) { + this.rows = rows; + } + } + + private static final class DatagramDecoder { + private final PacketReader reader; + + private DatagramDecoder(byte[] packet) { + this.reader = new PacketReader(packet); + } + + private List decode() { + Assert.assertEquals(MAGIC_MESSAGE, reader.readIntLE()); + Assert.assertEquals(VERSION_1, reader.readByte()); + reader.readByte(); + int tableCount = reader.readUnsignedShortLE(); + int payloadLength = reader.readIntLE(); + Assert.assertEquals(reader.length() - HEADER_SIZE, payloadLength); + + ArrayList rows = new ArrayList<>(); + for (int table = 0; table < tableCount; table++) { + rows.addAll(decodeTable()); + } + Assert.assertEquals(reader.length(), reader.position()); + return rows; + } + + private List decodeTable() { + String tableName = reader.readString(); + int rowCount = (int) reader.readVarint(); + int columnCount = (int) reader.readVarint(); + Assert.assertEquals(SCHEMA_MODE_FULL, reader.readByte()); + + QwpColumnDef[] defs = new QwpColumnDef[columnCount]; + for (int i = 0; i < columnCount; i++) { + defs[i] = new QwpColumnDef(reader.readString(), reader.readByte()); + } + + ColumnValues[] columns = new ColumnValues[columnCount]; + for (int i = 0; i < columnCount; i++) { + columns[i] = decodeColumn(defs[i], rowCount); + } + + ArrayList rows = new ArrayList<>(rowCount); + for (int row = 0; row < rowCount; row++) { + LinkedHashMap values = new LinkedHashMap<>(); + for (int col = 0; col < columnCount; col++) { + values.put(defs[col].getName(), columns[col].rows[row]); + } + rows.add(new DecodedRow(tableName, values)); + } + return rows; + } + + private ColumnValues decodeColumn(QwpColumnDef def, int rowCount) { + boolean[] nulls = def.isNullable() ? reader.readNullBitmap(rowCount) : new boolean[rowCount]; + int valueCount = rowCount - countNulls(nulls); + Object[] values = new Object[rowCount]; + + switch (def.getTypeCode()) { + case TYPE_BOOLEAN: + decodeBooleans(values, nulls, valueCount); + break; + case TYPE_DECIMAL64: + decodeDecimals(values, nulls, valueCount, 8); + break; + case TYPE_DECIMAL128: + decodeDecimals(values, nulls, valueCount, 16); + break; + case TYPE_DECIMAL256: + decodeDecimals(values, nulls, valueCount, 32); + break; + case TYPE_DOUBLE: + decodeDoubles(values, nulls, valueCount); + break; + case TYPE_DOUBLE_ARRAY: + decodeDoubleArrays(values, nulls, valueCount); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + decodeLongs(values, nulls, valueCount); + break; + case TYPE_LONG_ARRAY: + decodeLongArrays(values, nulls, valueCount); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + decodeStrings(values, nulls, valueCount); + break; + case TYPE_SYMBOL: + decodeSymbols(values, nulls, valueCount); + break; + default: + throw new AssertionError("Unsupported test decoder type: " + def.getTypeCode()); + } + + return new ColumnValues(values); + } + + private void decodeBooleans(Object[] values, boolean[] nulls, int valueCount) { + byte[] packed = reader.readBytes((valueCount + 7) / 8); + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + int byteIndex = valueIndex >>> 3; + int bitIndex = valueIndex & 7; + values[row] = (packed[byteIndex] & (1 << bitIndex)) != 0; + valueIndex++; + } + } + } + + private void decodeDecimals(Object[] values, boolean[] nulls, int valueCount, int width) { + int scale = reader.readByte(); + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + byte[] bytes = reader.readBytes(width); + values[row] = new BigDecimal(new BigInteger(bytes), scale); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeDoubleArrays(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + continue; + } + int dims = reader.readUnsignedByte(); + ArrayList shape = new ArrayList<>(dims); + int elementCount = 1; + for (int d = 0; d < dims; d++) { + int dim = reader.readIntLE(); + shape.add(dim); + elementCount = Math.multiplyExact(elementCount, dim); + } + ArrayList elements = new ArrayList<>(elementCount); + for (int i = 0; i < elementCount; i++) { + elements.add(reader.readDoubleLE()); + } + values[row] = new DoubleArrayValue(shape, elements); + valueIndex++; + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeDoubles(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + values[row] = reader.readDoubleLE(); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeLongArrays(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + continue; + } + int dims = reader.readUnsignedByte(); + ArrayList shape = new ArrayList<>(dims); + int elementCount = 1; + for (int d = 0; d < dims; d++) { + int dim = reader.readIntLE(); + shape.add(dim); + elementCount = Math.multiplyExact(elementCount, dim); + } + ArrayList elements = new ArrayList<>(elementCount); + for (int i = 0; i < elementCount; i++) { + elements.add(reader.readLongLE()); + } + values[row] = new LongArrayValue(shape, elements); + valueIndex++; + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeLongs(Object[] values, boolean[] nulls, int valueCount) { + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + values[row] = reader.readLongLE(); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeStrings(Object[] values, boolean[] nulls, int valueCount) { + int[] offsets = new int[valueCount + 1]; + for (int i = 0; i <= valueCount; i++) { + offsets[i] = reader.readIntLE(); + } + byte[] data = reader.readBytes(offsets[valueCount]); + + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + int start = offsets[valueIndex]; + int end = offsets[valueIndex + 1]; + values[row] = new String(data, start, end - start, StandardCharsets.UTF_8); + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private void decodeSymbols(Object[] values, boolean[] nulls, int valueCount) { + int dictSize = (int) reader.readVarint(); + String[] dict = new String[dictSize]; + for (int i = 0; i < dictSize; i++) { + dict[i] = reader.readString(); + } + + int valueIndex = 0; + for (int row = 0; row < values.length; row++) { + if (nulls[row]) { + values[row] = null; + } else { + int index = (int) reader.readVarint(); + values[row] = dict[index]; + valueIndex++; + } + } + Assert.assertEquals(valueCount, valueIndex); + } + + private static int countNulls(boolean[] nulls) { + int count = 0; + for (boolean value : nulls) { + if (value) { + count++; + } + } + return count; + } + } + + private static final class DecodedRow { + private final String table; + private final LinkedHashMap values; + + private DecodedRow(String table, LinkedHashMap values) { + this.table = table; + this.values = values; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DecodedRow that = (DecodedRow) o; + return Objects.equals(table, that.table) && Objects.equals(values, that.values); + } + + @Override + public int hashCode() { + return Objects.hash(table, values); + } + + @Override + public String toString() { + return "DecodedRow{" + + "table='" + table + '\'' + + ", values=" + values + + '}'; + } + } + + private static final class DoubleArrayValue { + private final List shape; + private final List values; + + private DoubleArrayValue(List shape, List values) { + this.shape = shape; + this.values = values; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DoubleArrayValue that = (DoubleArrayValue) o; + return Objects.equals(shape, that.shape) && Objects.equals(values, that.values); + } + + @Override + public int hashCode() { + return Objects.hash(shape, values); + } + + @Override + public String toString() { + return "DoubleArrayValue{" + + "shape=" + shape + + ", values=" + values + + '}'; + } + } + + private static final class LongArrayValue { + private final List shape; + private final List values; + + private LongArrayValue(List shape, List values) { + this.shape = shape; + this.values = values; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LongArrayValue that = (LongArrayValue) o; + return Objects.equals(shape, that.shape) && Objects.equals(values, that.values); + } + + @Override + public int hashCode() { + return Objects.hash(shape, values); + } + + @Override + public String toString() { + return "LongArrayValue{" + + "shape=" + shape + + ", values=" + values + + '}'; + } + } + + private static final class PacketReader { + private final byte[] data; + private int position; + + private PacketReader(byte[] data) { + this.data = data; + } + + private byte readByte() { + return data[position++]; + } + + private byte[] readBytes(int len) { + byte[] bytes = Arrays.copyOfRange(data, position, position + len); + position += len; + return bytes; + } + + private boolean[] readNullBitmap(int rowCount) { + byte[] bitmap = readBytes((rowCount + 7) / 8); + boolean[] nulls = new boolean[rowCount]; + for (int row = 0; row < rowCount; row++) { + int byteIndex = row >>> 3; + int bitIndex = row & 7; + nulls[row] = (bitmap[byteIndex] & (1 << bitIndex)) != 0; + } + return nulls; + } + + private double readDoubleLE() { + double value = Unsafe.getUnsafe().getDouble(data, Unsafe.BYTE_OFFSET + position); + position += Double.BYTES; + return value; + } + + private int readIntLE() { + int value = Unsafe.byteArrayGetInt(data, position); + position += Integer.BYTES; + return value; + } + + private long readLongLE() { + long value = Unsafe.byteArrayGetLong(data, position); + position += Long.BYTES; + return value; + } + + private String readString() { + int len = (int) readVarint(); + if (len == 0) { + return ""; + } + String value = new String(data, position, len, StandardCharsets.UTF_8); + position += len; + return value; + } + + private int readUnsignedByte() { + return readByte() & 0xff; + } + + private int readUnsignedShortLE() { + int value = Unsafe.byteArrayGetShort(data, position) & 0xffff; + position += Short.BYTES; + return value; + } + + private long readVarint() { + long value = 0; + int shift = 0; + while (true) { + int b = readUnsignedByte(); + value |= (long) (b & 0x7f) << shift; + if ((b & 0x80) == 0) { + return value; + } + shift += 7; + if (shift > 63) { + throw new AssertionError("varint too long"); + } + } + } + + private int length() { + return data.length; + } + + private int position() { + return position; + } + } + + private static final class RunResult { + private final List lengths; + private final List packets; + private final int sendCount; + + private RunResult(List packets, List lengths, int sendCount) { + this.packets = new ArrayList<>(packets); + this.lengths = new ArrayList<>(lengths); + this.sendCount = sendCount; + } + } + + private static final class ScenarioRow { + private final DecodedRow expected; + private final ThrowingConsumer writer; + + private ScenarioRow(DecodedRow expected, ThrowingConsumer writer) { + this.expected = expected; + this.writer = writer; + } + } +} From ff163853e69719cf31527a743275c8e98c64c775 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 6 Mar 2026 10:59:22 +0100 Subject: [PATCH 154/230] Add WebSocket auth header support for QWP ingestion WebSocketClient.upgrade() now accepts an optional Authorization header, which is included in the HTTP upgrade request. QwpWebSocketSender connect/connectAsync factory methods accept an authorizationHeader parameter and pass it through to upgrade(). Sender.Builder builds Basic or Bearer headers from httpUsernamePassword/httpToken and wires them to the WebSocket sender. The config-string parser also accepts token and username/password for ws::/wss:: schemas. Co-Authored-By: Claude Opus 4.6 --- .../main/java/io/questdb/client/Sender.java | 34 +++++-- .../cutlass/http/client/WebSocketClient.java | 28 +++++- .../qwp/client/QwpWebSocketSender.java | 99 +++++++++++++------ .../LineSenderBuilderWebSocketTest.java | 58 ++++------- 4 files changed, 138 insertions(+), 81 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index c815bf7..8007407 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -52,7 +52,9 @@ import javax.security.auth.DestroyFailedException; import java.io.Closeable; +import java.nio.charset.StandardCharsets; import java.security.PrivateKey; +import java.util.Base64; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; @@ -867,6 +869,8 @@ public Sender build() { : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis); int actualInFlightWindowSize = inFlightWindowSize == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_IN_FLIGHT_WINDOW_SIZE : inFlightWindowSize; + String wsAuthHeader = buildWebSocketAuthHeader(); + if (asyncMode) { return QwpWebSocketSender.connectAsync( hosts.getQuick(0), @@ -875,7 +879,8 @@ public Sender build() { actualAutoFlushRows, actualAutoFlushBytes, actualAutoFlushIntervalNanos, - actualInFlightWindowSize + actualInFlightWindowSize, + wsAuthHeader ); } else { return QwpWebSocketSender.connect( @@ -884,7 +889,8 @@ public Sender build() { tlsEnabled, actualAutoFlushRows, actualAutoFlushBytes, - actualAutoFlushIntervalNanos + actualAutoFlushIntervalNanos, + wsAuthHeader ); } } @@ -1548,10 +1554,8 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { if (protocol == PROTOCOL_TCP) { tcpToken = sink.toString(); // will configure later, we need to know a keyId first - } else if (protocol == PROTOCOL_HTTP) { - httpToken(sink.toString()); } else { - throw new LineSenderException("token is not supported for WebSocket protocol"); + httpToken(sink.toString()); } } else if (Chars.equals("retry_timeout", sink)) { pos = getValue(configurationString, pos, sink, "retry_timeout"); @@ -1657,11 +1661,11 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else if (trustStorePassword != null) { throw new LineSenderException("tls_roots_password was configured, but tls_roots is missing"); } - if (protocol == PROTOCOL_HTTP) { + if (protocol == PROTOCOL_HTTP || protocol == PROTOCOL_WEBSOCKET) { if (user != null) { httpUsernamePassword(user, password); } else if (password != null) { - throw new LineSenderException("HTTP password is configured, but username is missing"); + throw new LineSenderException("password is configured, but username is missing"); } } else { if (user != null) { @@ -1758,9 +1762,8 @@ private void validateParameters() { if (privateKey != null) { throw new LineSenderException("TCP authentication is not supported for WebSocket protocol"); } - if (httpToken != null || username != null || password != null) { - // TODO: WebSocket auth not yet implemented - throw new LineSenderException("Authentication is not yet supported for WebSocket protocol"); + if (httpToken != null && (username != null || password != null)) { + throw new LineSenderException("cannot use both token and username/password authentication"); } if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { throw new LineSenderException("in-flight window size requires async mode"); @@ -1792,6 +1795,17 @@ private void validateParameters() { } } + private String buildWebSocketAuthHeader() { + if (username != null && password != null) { + String credentials = username + ":" + password; + return "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + } + if (httpToken != null) { + return "Bearer " + httpToken; + } + return null; + } + private void websocket() { if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol was already configured ") diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 84c4329..e57dbe0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -383,10 +383,11 @@ public boolean tryReceiveFrame(WebSocketFrameHandler handler) { /** * Performs WebSocket upgrade handshake. * - * @param path the WebSocket endpoint path (e.g., "/ws") - * @param timeout timeout in milliseconds + * @param path the WebSocket endpoint path (e.g., "/ws") + * @param timeout timeout in milliseconds + * @param authorizationHeader the Authorization header value (e.g., "Basic ..."), or null */ - public void upgrade(CharSequence path, int timeout) { + public void upgrade(CharSequence path, int timeout, CharSequence authorizationHeader) { if (closed) { throw new HttpClientException("WebSocket client is closed"); } @@ -422,6 +423,11 @@ public void upgrade(CharSequence path, int timeout) { sendBuffer.putAscii(handshakeKey); sendBuffer.putAscii("\r\n"); sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n"); + if (authorizationHeader != null) { + sendBuffer.putAscii("Authorization: "); + sendBuffer.putAscii(authorizationHeader); + sendBuffer.putAscii("\r\n"); + } sendBuffer.putAscii("\r\n"); // Send request @@ -441,7 +447,21 @@ public void upgrade(CharSequence path, int timeout) { * Performs upgrade with default timeout. */ public void upgrade(CharSequence path) { - upgrade(path, defaultTimeout); + upgrade(path, defaultTimeout, null); + } + + /** + * Performs upgrade with default timeout and authorization header. + */ + public void upgrade(CharSequence path, CharSequence authorizationHeader) { + upgrade(path, defaultTimeout, authorizationHeader); + } + + /** + * Performs upgrade without authorization header. + */ + public void upgrade(CharSequence path, int timeout) { + upgrade(path, timeout, null); } private static String computeAcceptKey(String key) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index cdfbe06..2cfb689 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -117,6 +117,7 @@ public class QwpWebSocketSender implements Sender { private static final String WRITE_PATH = "/write/v4"; private final AckFrameHandler ackHandler = new AckFrameHandler(this); private final WebSocketResponse ackResponse = new WebSocketResponse(); + private final String authorizationHeader; private final int autoFlushBytes; private final long autoFlushIntervalNanos; // Auto-flush configuration @@ -174,8 +175,10 @@ private QwpWebSocketSender( int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize + int inFlightWindowSize, + String authorizationHeader ) { + this.authorizationHeader = authorizationHeader; this.host = host; this.port = port; this.tlsEnabled = tlsEnabled; @@ -224,8 +227,9 @@ public static QwpWebSocketSender connect(String host, int port) { * @return connected sender */ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled) { - return connect(host, port, tlsEnabled, - DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS); + return connect( + host, port, tlsEnabled, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS + ); } /** @@ -240,13 +244,30 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @return connected sender */ - public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos) { + public static QwpWebSocketSender connect( + String host, + int port, + boolean tlsEnabled, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos + ) { + return connect(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, null); + } + + public static QwpWebSocketSender connect( + String host, + int port, + boolean tlsEnabled, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + String authorizationHeader + ) { QwpWebSocketSender sender = new QwpWebSocketSender( - host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, - autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - 1 // window=1 for sync behavior + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + 1, // window=1 for sync behavior + authorizationHeader ); sender.ensureConnected(); return sender; @@ -263,11 +284,17 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @return connected sender */ - public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos) { - return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - DEFAULT_IN_FLIGHT_WINDOW_SIZE); + public static QwpWebSocketSender connectAsync( + String host, + int port, + boolean tlsEnabled, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos + ) { + return connectAsync( + host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, DEFAULT_IN_FLIGHT_WINDOW_SIZE + ); } /** @@ -290,11 +317,24 @@ public static QwpWebSocketSender connectAsync( int autoFlushBytes, long autoFlushIntervalNanos, int inFlightWindowSize + ) { + return connectAsync( + host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, null + ); + } + + public static QwpWebSocketSender connectAsync( + String host, + int port, + boolean tlsEnabled, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + String authorizationHeader ) { QwpWebSocketSender sender = new QwpWebSocketSender( - host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, - autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - inFlightWindowSize + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, authorizationHeader ); sender.ensureConnected(); return sender; @@ -309,8 +349,9 @@ public static QwpWebSocketSender connectAsync( * @return connected sender */ public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { - return connectAsync(host, port, tlsEnabled, - DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS); + return connectAsync( + host, port, tlsEnabled, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS + ); } /** @@ -326,9 +367,7 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls */ public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { return new QwpWebSocketSender( - host, port, false, DEFAULT_BUFFER_SIZE, - DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, - inFlightWindowSize + host, port, false, DEFAULT_BUFFER_SIZE, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, inFlightWindowSize, null ); // Note: does NOT call ensureConnected() } @@ -345,13 +384,15 @@ public static QwpWebSocketSender createForTesting(String host, int port, int inF * @return unconnected sender */ public static QwpWebSocketSender createForTesting( - String host, int port, - int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize) { + String host, + int port, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize + ) { return new QwpWebSocketSender( - host, port, false, DEFAULT_BUFFER_SIZE, - autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - inFlightWindowSize + host, port, false, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, null ); // Note: does NOT call ensureConnected() } @@ -1012,7 +1053,7 @@ private void ensureConnected() { // Connect and upgrade to WebSocket try { client.connect(host, port); - client.upgrade(WRITE_PATH); + client.upgrade(WRITE_PATH, authorizationHeader); } catch (Exception e) { client.close(); client = null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index ae6480c..14d5aa7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -361,22 +361,11 @@ public void testHttpTimeout_notSupportedForWebSocket() { } @Test - public void testHttpToken_fails() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .httpToken("token"), - "not yet supported"); - } - - @Test - @Ignore("HTTP token authentication is not yet supported for WebSocket protocol") - public void testHttpToken_notYetSupported() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .httpToken("token"), - "not yet supported"); + public void testHttpToken_accepted() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpToken("token"); + Assert.assertNotNull(builder); } @Test @@ -616,22 +605,11 @@ public void testTlsValidationDisabled_butTlsNotEnabled_fails() { } @Test - public void testUsernamePassword_fails() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .httpUsernamePassword("user", "pass"), - "not yet supported"); - } - - @Test - @Ignore("Username/password authentication is not yet supported for WebSocket protocol") - public void testUsernamePassword_notYetSupported() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .httpUsernamePassword("user", "pass"), - "not yet supported"); + public void testUsernamePassword_accepted() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpUsernamePassword("user", "pass"); + Assert.assertNotNull(builder); } @Test @@ -669,15 +647,19 @@ public void testWsConfigString_uppercaseNotSupported() { } @Test - @Ignore("Token authentication in ws config string is not yet supported") - public void testWsConfigString_withToken_notYetSupported() { - assertBadConfig("ws::addr=localhost:9000;token=mytoken;", "not yet supported"); + public void testWsConfigString_withToken() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";token=mytoken;", "connect", "Failed"); + }); } @Test - @Ignore("Username/password in ws config string is not yet supported") - public void testWsConfigString_withUsernamePassword_notYetSupported() { - assertBadConfig("ws::addr=localhost:9000;username=user;password=pass;", "not yet supported"); + public void testWsConfigString_withUsernamePassword() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";username=user;password=pass;", "connect", "Failed"); + }); } @Test From 34b4a115f23806f2bd941804c5f8aec4c5a1905f Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 11:10:45 +0100 Subject: [PATCH 155/230] optimize column lookup performance --- .../cutlass/qwp/client/QwpUdpSender.java | 9 +- .../cutlass/qwp/protocol/QwpTableBuffer.java | 75 +++++--- .../cutlass/qwp/client/QwpUdpSenderTest.java | 74 ++++++++ .../qwp/protocol/QwpTableBufferTest.java | 177 ++++++++++++++++++ 4 files changed, 303 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index c216f4b..07fd714 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -414,15 +414,18 @@ public Sender timestampColumn(CharSequence columnName, Instant value) { } private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean nullable) { - boolean exists = currentTableBuffer.hasColumn(name); - if (!exists && currentTableBuffer.getRowCount() > 0) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn(name, type); + if (col == null && currentTableBuffer.getRowCount() > 0) { if (currentRowColumnCount > 0) { throw new LineSenderException("schema change in middle of row is not supported"); } flushSingleTable(currentTableName, currentTableBuffer); + col = currentTableBuffer.getOrCreateColumn(name, type, nullable); } - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, type, nullable); + if (col == null) { + col = currentTableBuffer.getOrCreateColumn(name, type, nullable); + } syncSchemaEstimate(); return col; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 8676021..51089f0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -149,8 +149,14 @@ public int getColumnCount() { return columns.size(); } - public boolean hasColumn(CharSequence name) { - return columnNameToIndex.get(name) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + /** + * Returns an existing column with the given name and type, or {@code null} if absent. + *

    + * Uses the same sequential access optimization as {@link #getOrCreateColumn(CharSequence, byte, boolean)}. + * When the next expected column is accessed in order, the internal cursor advances without a hash lookup. + */ + public ColumnBuffer getExistingColumn(CharSequence name, byte type) { + return lookupColumn(name, type); } /** @@ -175,36 +181,15 @@ public QwpColumnDef[] getColumnDefs() { * order every row: a sequential cursor avoids hash map lookups entirely. */ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean nullable) { - // Fast path: predict next column in sequence - int n = columns.size(); - if (columnAccessCursor < n) { - ColumnBuffer candidate = fastColumns[columnAccessCursor]; - if (Chars.equals(candidate.name, name)) { - columnAccessCursor++; - if (candidate.type != type) { - throw new LineSenderException( - "Column type mismatch for column '" + name + "': columnType=" - + candidate.type + ", sentType=" + type - ); - } - return candidate; - } - } - - // Slow path: hash map lookup - int idx = columnNameToIndex.get(name); - if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) { - ColumnBuffer existing = columns.get(idx); - if (existing.type != type) { - throw new LineSenderException( - "Column type mismatch for column '" + name + "': columnType=" - + existing.type + ", sentType=" + type - ); - } + ColumnBuffer existing = lookupColumn(name, type); + if (existing != null) { return existing; } - // Create new column + return createColumn(name, type, nullable); + } + + private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) { ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); int index = columns.size(); columns.add(col); @@ -224,6 +209,38 @@ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean null return col; } + private ColumnBuffer lookupColumn(CharSequence name, byte type) { + // Fast path: predict next column in sequence + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + if (Chars.equals(candidate.name, name)) { + columnAccessCursor++; + assertColumnType(name, type, candidate); + return candidate; + } + } + + // Slow path: hash map lookup + int idx = columnNameToIndex.get(name); + if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + ColumnBuffer existing = columns.get(idx); + assertColumnType(name, type, existing); + return existing; + } + + return null; + } + + private static void assertColumnType(CharSequence name, byte type, ColumnBuffer column) { + if (column.type != type) { + throw new LineSenderException( + "Column type mismatch for column '" + name + "': columnType=" + + column.type + ", sentType=" + type + ); + } + } + /** * Returns the number of rows buffered. */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 4562dea..9c40db9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -260,6 +260,45 @@ public void testBoundedSenderMixedTypesPreservesRowsAndPacketLimit() throws Exce }); } + @Test + public void testBoundedSenderOutOfOrderExistingColumnsPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("order", sender -> sender.table("order") + .longColumn("a", 1) + .stringColumn("b", "x") + .symbol("c", "alpha") + .atNow(), + "a", 1L, + "b", "x", + "c", "alpha"), + row("order", sender -> sender.table("order") + .symbol("c", "beta") + .stringColumn("b", "y") + .longColumn("a", 2) + .atNow(), + "a", 2L, + "b", "y", + "c", "beta"), + row("order", sender -> sender.table("order") + .stringColumn("b", "z") + .longColumn("a", 3) + .symbol("c", null) + .atNow(), + "a", 3L, + "b", "z", + "c", null) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + @Test public void testMixingAtNowAndAtMicrosAfterCommittedRowsThrowsSchemaChange() throws Exception { assertMemoryLeak(() -> { @@ -478,6 +517,41 @@ public void testSchemaChangeWithCommittedRowsFlushesImmediately() throws Excepti }); } + @Test + public void testSchemaChangeAfterOutOfOrderExistingColumnsPreservesRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("schema", sender -> sender.table("schema") + .longColumn("a", 1) + .stringColumn("b", "x") + .atNow(), + "a", 1L, + "b", "x"), + row("schema", sender -> sender.table("schema") + .symbol("c", "new") + .stringColumn("b", "y") + .longColumn("a", 2) + .atNow(), + "a", 2L, + "b", "y", + "c", "new"), + row("schema", sender -> sender.table("schema") + .symbol("c", "next") + .longColumn("a", 3) + .stringColumn("b", "z") + .atNow(), + "a", 3L, + "b", "z", + "c", "next") + ); + + RunResult result = runScenario(rows, 1024 * 1024); + + Assert.assertEquals(2, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + @Test public void testSymbolBoundary16383To16384PreservesRowsAndPacketLimit() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index ec3ef6d..9d825a9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -701,6 +701,183 @@ public void testLongArrayShrinkingSize() throws Exception { }); } + @Test + public void testGetExistingColumnReturnsOrderedColumnsAcrossRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + + assertSame(colA, existingA); + assertSame(colB, existingB); + + existingA.addLong(2); + existingB.addString("y"); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(2, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnReturnsOutOfOrderColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + + assertSame(colB, existingB); + assertSame(colA, existingA); + + existingB.addString("y"); + existingA.addLong(2); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(2, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnReturnsNullWithoutCreatingColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + colA.addLong(1); + table.nextRow(); + + assertNull(table.getExistingColumn("missing", QwpConstants.TYPE_STRING)); + assertEquals(1, table.getColumnCount()); + + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + assertNotNull(colB); + assertEquals(2, table.getColumnCount()); + } + }); + } + + @Test + public void testGetExistingColumnTypeMismatchOnOrderedPathThrows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + table.nextRow(); + + try { + table.getExistingColumn("a", QwpConstants.TYPE_STRING); + fail("Expected LineSenderException for ordered-path type mismatch"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Column type mismatch")); + assertTrue(e.getMessage().contains("column 'a'")); + } + } + }); + } + + @Test + public void testGetExistingColumnTypeMismatchOnHashPathThrows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + try { + table.getExistingColumn("b", QwpConstants.TYPE_LONG); + fail("Expected LineSenderException for hash-path type mismatch"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Column type mismatch")); + assertTrue(e.getMessage().contains("column 'b'")); + } + } + }); + } + + @Test + public void testGetExistingColumnWorksAfterReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + table.reset(); + + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + + assertSame(colA, existingA); + assertSame(colB, existingB); + + existingA.addLong(2); + existingB.addString("y"); + table.nextRow(); + + assertEquals(1, table.getRowCount()); + assertEquals(1, colA.getSize()); + assertEquals(1, colA.getValueCount()); + assertEquals(1, colB.getSize()); + assertEquals(1, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnWorksForLateAddedColumnAfterCancelRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + table.nextRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(2); + QwpTableBuffer.ColumnBuffer late = table.getOrCreateColumn("late", QwpConstants.TYPE_STRING, true); + late.addString("stale"); + table.cancelCurrentRow(); + + QwpTableBuffer.ColumnBuffer existingLate = table.getExistingColumn("late", QwpConstants.TYPE_STRING); + assertSame(late, existingLate); + assertEquals(0, existingLate.getSize()); + assertEquals(0, existingLate.getValueCount()); + + table.getExistingColumn("a", QwpConstants.TYPE_LONG).addLong(2); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, existingLate.getSize()); + assertEquals(0, existingLate.getValueCount()); + assertTrue(existingLate.isNull(0)); + assertTrue(existingLate.isNull(1)); + } + }); + } + /** * Simulates the encoder's walk over array data — the same logic as * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat From a32a77123cab3f3e2df47d4d6a383f8a8e29ae01 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 11:36:02 +0100 Subject: [PATCH 156/230] optimize datagram size tracking and null padding --- .../cutlass/qwp/client/QwpUdpSender.java | 79 ++++++++++++++----- .../cutlass/qwp/protocol/QwpTableBuffer.java | 18 +++++ .../qwp/protocol/QwpTableBufferTest.java | 24 ++++++ 3 files changed, 101 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 07fd714..cbf8c86 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -84,6 +84,7 @@ public class QwpUdpSender implements Sender { private final UdpLineChannel channel; private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private final int maxDatagramSize; + private final boolean trackDatagramEstimate; private final ObjList rowJournal = new ObjList<>(); private final CharSequenceObjHashMap tableBuffers; @@ -96,6 +97,8 @@ public class QwpUdpSender implements Sender { private QwpTableBuffer currentTableBuffer; private String currentTableName; private int estimateColumnCount; + private QwpTableBuffer.ColumnBuffer[] missingColumns = new QwpTableBuffer.ColumnBuffer[8]; + private int missingColumnCount; private int rowJournalSize; private long runningEstimate; @@ -107,6 +110,7 @@ public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int this.channel = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); this.tableBuffers = new CharSequenceObjHashMap<>(); this.maxDatagramSize = maxDatagramSize; + this.trackDatagramEstimate = maxDatagramSize > 0; } @Override @@ -374,7 +378,7 @@ public Sender table(CharSequence tableName) { return this; } ensureNoInProgressRow("switch tables"); - if (maxDatagramSize > 0 && currentTableBuffer != null && currentTableBuffer.getRowCount() > 0) { + if (trackDatagramEstimate && currentTableBuffer != null && currentTableBuffer.getRowCount() > 0) { flushSingleTable(currentTableName, currentTableBuffer); } cachedTimestampColumn = null; @@ -440,7 +444,7 @@ private void appendBooleanColumn(CharSequence name, boolean value, boolean addJo applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_BOOL; e.name = col.getName(); @@ -458,7 +462,7 @@ private void appendDecimal128Column(CharSequence name, Decimal128 value, boolean applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL128; e.name = col.getName(); @@ -476,7 +480,7 @@ private void appendDecimal256Column(CharSequence name, Decimal256 value, boolean applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL256; e.name = col.getName(); @@ -494,7 +498,7 @@ private void appendDecimal64Column(CharSequence name, Decimal64 value, boolean a applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL64; e.name = col.getName(); @@ -527,7 +531,7 @@ private void appendDesignatedTimestamp(long value, boolean nanos, boolean addJou long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS; e.longValue = value; @@ -563,7 +567,7 @@ private void appendDoubleArrayColumn(CharSequence name, Object value, boolean ad applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE_ARRAY; e.name = col.getName(); @@ -581,7 +585,7 @@ private void appendDoubleColumn(CharSequence name, double value, boolean addJour applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE; e.name = col.getName(); @@ -618,7 +622,7 @@ private void appendLongArrayColumn(CharSequence name, Object value, boolean addJ applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG_ARRAY; e.name = col.getName(); @@ -636,7 +640,7 @@ private void appendLongColumn(CharSequence name, long value, boolean addJournal) applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG; e.name = col.getName(); @@ -656,7 +660,7 @@ private void appendStringColumn(CharSequence name, CharSequence value, boolean a applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_STRING; e.name = col.getName(); @@ -675,7 +679,7 @@ private void appendSymbolColumn(CharSequence name, CharSequence value, boolean a applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_SYMBOL; e.name = col.getName(); @@ -693,7 +697,7 @@ private void appendTimestampColumn(CharSequence name, byte type, long value, byt applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && maxDatagramSize > 0) { + if (addJournal && trackDatagramEstimate) { ColumnEntry e = nextJournalEntry(); e.kind = journalKind; e.name = col.getName(); @@ -701,7 +705,8 @@ private void appendTimestampColumn(CharSequence name, byte type, long value, byt } } - private void applyRowPaddingEstimate(int targetRows) { + private void collectMissingColumns(int targetRows) { + missingColumnCount = 0; for (int i = 0, n = currentTableBuffer.getColumnCount(); i < n; i++) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); int sizeBefore = col.getSize(); @@ -710,6 +715,13 @@ private void applyRowPaddingEstimate(int targetRows) { continue; } + ensureMissingColumnCapacity(missingColumnCount + 1); + missingColumns[missingColumnCount++] = col; + + if (!trackDatagramEstimate) { + continue; + } + if (col.isNullable()) { runningEstimate += bitmapBytes(sizeBefore + missing) - bitmapBytes(sizeBefore); continue; @@ -721,6 +733,9 @@ private void applyRowPaddingEstimate(int targetRows) { } private void applyValueEstimate(QwpTableBuffer.ColumnBuffer col, int sizeBefore, int sizeAfter, long payloadDelta) { + if (!trackDatagramEstimate) { + return; + } runningEstimate += payloadDelta; if (col.isNullable()) { runningEstimate += bitmapBytes(sizeAfter) - bitmapBytes(sizeBefore); @@ -761,16 +776,19 @@ private void commitCurrentRow() { } int targetRows = currentTableBuffer.getRowCount() + 1; - applyRowPaddingEstimate(targetRows); + collectMissingColumns(targetRows); - if (maxDatagramSize > 0) { + if (trackDatagramEstimate) { maybeAutoFlush(); } - currentTableBuffer.nextRow(); - committedEstimate = runningEstimate; - committedEstimateColumnCount = estimateColumnCount; + currentTableBuffer.nextRow(missingColumns, missingColumnCount); + if (trackDatagramEstimate) { + committedEstimate = runningEstimate; + committedEstimateColumnCount = estimateColumnCount; + } currentRowColumnCount = 0; + missingColumnCount = 0; rowJournalSize = 0; } @@ -818,6 +836,21 @@ private long estimateArrayValueSize(LongArray array) { return arraySizeCounter.size; } + private void ensureMissingColumnCapacity(int required) { + if (required <= missingColumns.length) { + return; + } + + int newCapacity = missingColumns.length; + while (newCapacity < required) { + newCapacity *= 2; + } + + QwpTableBuffer.ColumnBuffer[] newArr = new QwpTableBuffer.ColumnBuffer[newCapacity]; + System.arraycopy(missingColumns, 0, newArr, 0, missingColumnCount); + missingColumns = newArr; + } + private long estimateSymbolPayloadDelta( QwpTableBuffer.ColumnBuffer col, int valueCountBefore, @@ -908,7 +941,7 @@ private void maybeAutoFlush() { flushSingleTable(currentTableName, currentTableBuffer); replayRowJournal(); - applyRowPaddingEstimate(currentTableBuffer.getRowCount() + 1); + collectMissingColumns(currentTableBuffer.getRowCount() + 1); if (runningEstimate > maxDatagramSize) { throw singleRowTooLarge(runningEstimate); @@ -983,6 +1016,7 @@ private void resetEstimateState() { estimateColumnCount = 0; committedEstimateColumnCount = 0; currentRowColumnCount = 0; + missingColumnCount = 0; } private boolean hasInProgressRow() { @@ -993,6 +1027,7 @@ private void rollbackEstimateToCommitted() { runningEstimate = committedEstimate; estimateColumnCount = committedEstimateColumnCount; currentRowColumnCount = 0; + missingColumnCount = 0; } private LineSenderException singleRowTooLarge(long estimate) { @@ -1003,6 +1038,10 @@ private LineSenderException singleRowTooLarge(long estimate) { } private void syncSchemaEstimate() { + if (!trackDatagramEstimate) { + return; + } + int newColumnCount = currentTableBuffer.getColumnCount(); if (newColumnCount == estimateColumnCount) { return; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 51089f0..b6c2128 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -300,6 +300,24 @@ public void nextRow() { committedColumnCount = columns.size(); } + /** + * Advances to the next row using a prepared list of columns that need null padding. + *

    + * This avoids rescanning every column when the caller has already identified + * which columns were omitted in the current row. + */ + public void nextRow(ColumnBuffer[] missingColumns, int missingColumnCount) { + columnAccessCursor = 0; + for (int i = 0; i < missingColumnCount; i++) { + ColumnBuffer col = missingColumns[i]; + while (col.size < rowCount + 1) { + col.addNull(); + } + } + rowCount++; + committedColumnCount = columns.size(); + } + /** * Resets the buffer for reuse. Keeps column definitions and allocated memory. */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 9d825a9..8b61fb6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -292,6 +292,30 @@ public void testCancelRowTruncatesLateAddedColumnWhenSizeEqualsRowCount() throws }); } + @Test + public void testNextRowWithPreparedMissingColumnsPadsListedColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + + colA.addLong(10); + colB.addString("x"); + table.nextRow(); + + colA.addLong(20); + table.nextRow(new QwpTableBuffer.ColumnBuffer[]{colB}, 1); + + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(1, colB.getValueCount()); + assertFalse(colB.isNull(0)); + assertTrue(colB.isNull(1)); + } + }); + } + @Test public void testCancelRowResetsDecimalScaleOnLateAddedColumn() throws Exception { assertMemoryLeak(() -> { From 0cee86f188441f2f37d81955f2b9efc936e9f66e Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 11:38:25 +0100 Subject: [PATCH 157/230] add tests for sender behavior with omitted columns and packet limits --- .../cutlass/qwp/client/QwpUdpSenderTest.java | 102 ++++++++++++++++++ .../qwp/protocol/QwpTableBufferTest.java | 9 +- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 9c40db9..12c9492 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -136,6 +136,74 @@ public void testBoundedSenderMixedNullablePaddingPreservesRowsAndPacketLimit() t }); } + @Test + public void testUnboundedSenderOmittedNullableAndNonNullableColumnsPreservesRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 1) + .stringColumn("s", "alpha") + .symbol("sym", "one") + .atNow(), + "x", 1L, + "s", "alpha", + "sym", "one"), + row("t", sender -> sender.table("t") + .stringColumn("s", "beta") + .atNow(), + "x", Long.MIN_VALUE, + "s", "beta", + "sym", null), + row("t", sender -> sender.table("t") + .longColumn("x", 3) + .atNow(), + "x", 3L, + "s", null, + "sym", null) + ); + + RunResult result = runScenario(rows, 0); + + Assert.assertEquals(1, result.sendCount); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + + @Test + public void testBoundedSenderOmittedNonNullableColumnsPreservesRowsAndPacketLimit() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String beta = repeat('b', 192); + String omega = repeat('z', 256); + List rows = Arrays.asList( + row("mix", sender -> sender.table("mix") + .longColumn("x", 1) + .stringColumn("msg", alpha) + .atNow(), + "x", 1L, + "msg", alpha), + row("mix", sender -> sender.table("mix") + .stringColumn("msg", beta) + .atNow(), + "x", Long.MIN_VALUE, + "msg", beta), + row("mix", sender -> sender.table("mix") + .longColumn("x", 3) + .stringColumn("msg", omega) + .atNow(), + "x", 3L, + "msg", omega) + ); + + int maxDatagramSize = fullPacketSize(rows) - 1; + RunResult result = runScenario(rows, maxDatagramSize); + + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + @Test public void testBoundedSenderArrayReplayPreservesRowsAndPacketLimit() throws Exception { assertMemoryLeak(() -> { @@ -552,6 +620,40 @@ public void testSchemaChangeAfterOutOfOrderExistingColumnsPreservesRows() throws }); } + @Test + public void testBoundedSenderSchemaFlushThenOmittedNullableColumnsPreservesRows() throws Exception { + assertMemoryLeak(() -> { + List rows = Arrays.asList( + row("schema", sender -> sender.table("schema") + .longColumn("x", 1) + .stringColumn("s", "alpha") + .atNow(), + "x", 1L, + "s", "alpha"), + row("schema", sender -> sender.table("schema") + .symbol("sym", "new") + .longColumn("x", 2) + .stringColumn("s", "beta") + .atNow(), + "sym", "new", + "x", 2L, + "s", "beta"), + row("schema", sender -> sender.table("schema") + .longColumn("x", 3) + .atNow(), + "sym", null, + "x", 3L, + "s", null) + ); + + RunResult result = runScenario(rows, 1024 * 1024); + + Assert.assertEquals(2, result.sendCount); + assertPacketsWithinLimit(result, 1024 * 1024); + assertRowsEqual(expectedRows(rows), decodeRows(result.packets)); + }); + } + @Test public void testSymbolBoundary16383To16384PreservesRowsAndPacketLimit() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 8b61fb6..a04e0b5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal64; import org.junit.Test; @@ -298,13 +299,15 @@ public void testNextRowWithPreparedMissingColumnsPadsListedColumns() throws Exce try (QwpTableBuffer table = new QwpTableBuffer("test")) { QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_LONG, false); colA.addLong(10); colB.addString("x"); + colC.addLong(100); table.nextRow(); colA.addLong(20); - table.nextRow(new QwpTableBuffer.ColumnBuffer[]{colB}, 1); + table.nextRow(new QwpTableBuffer.ColumnBuffer[]{colB, colC}, 2); assertEquals(2, colA.getSize()); assertEquals(2, colA.getValueCount()); @@ -312,6 +315,10 @@ public void testNextRowWithPreparedMissingColumnsPadsListedColumns() throws Exce assertEquals(1, colB.getValueCount()); assertFalse(colB.isNull(0)); assertTrue(colB.isNull(1)); + assertEquals(2, colC.getSize()); + assertEquals(2, colC.getValueCount()); + assertEquals(100L, Unsafe.getUnsafe().getLong(colC.getDataAddress())); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(colC.getDataAddress() + Long.BYTES)); } }); } From 8bc30e2fed5e575817b8a27d7d5fb737d83ac033 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 6 Mar 2026 12:36:52 +0100 Subject: [PATCH 158/230] Fix asserted error message --- .../client/test/cutlass/line/LineSenderBuilderTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java index 031f204..9feca2b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -27,7 +27,6 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.tools.TestUtils; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Test; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -151,8 +150,8 @@ public void testConfStringValidation() throws Exception { assertConfStrError("tcp::addr=localhost;token=foo;", "TCP token is configured, but user is missing"); assertConfStrError("http::addr=localhost;user=foo;", "password cannot be empty nor null"); assertConfStrError("http::addr=localhost;username=foo;", "password cannot be empty nor null"); - assertConfStrError("http::addr=localhost;pass=foo;", "HTTP password is configured, but username is missing"); - assertConfStrError("http::addr=localhost;password=foo;", "HTTP password is configured, but username is missing"); + assertConfStrError("http::addr=localhost;pass=foo;", "password is configured, but username is missing"); + assertConfStrError("http::addr=localhost;password=foo;", "password is configured, but username is missing"); assertConfStrError("tcp::addr=localhost;pass=foo;", "password is not supported for TCP protocol"); assertConfStrError("tcp::addr=localhost;password=foo;", "password is not supported for TCP protocol"); assertConfStrError("tcp::addr=localhost;retry_timeout=;", "retry_timeout cannot be empty"); From 8d32ed0fe7b65423aac9b8f6d6c33c1b398ba905 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 6 Mar 2026 12:50:56 +0100 Subject: [PATCH 159/230] Reject token in WebSocket config strings The config string parser accepted the `token` parameter for WebSocket protocols (ws/wss) and passed it through to httpToken(), which caused a confusing "Failed to connect" error instead of a clear validation message. Add an early check for PROTOCOL_WEBSOCKET in the token parsing branch to throw a descriptive error before attempting any connection. Co-Authored-By: Claude Opus 4.6 --- core/src/main/java/io/questdb/client/Sender.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 8007407..5c40de9 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1551,7 +1551,9 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } } else if (Chars.equals("token", sink)) { pos = getValue(configurationString, pos, sink, "token"); - if (protocol == PROTOCOL_TCP) { + if (protocol == PROTOCOL_WEBSOCKET) { + throw new LineSenderException("token is not supported for WebSocket protocol"); + } else if (protocol == PROTOCOL_TCP) { tcpToken = sink.toString(); // will configure later, we need to know a keyId first } else { From 80e65fb93d59c4a45bc6a2c4031fbad1c13beba8 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 6 Mar 2026 13:00:42 +0100 Subject: [PATCH 160/230] Fix error message asssertion --- .../cutlass/qwp/client/LineSenderBuilderWebSocketTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 14d5aa7..ba54fcc 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -29,11 +29,12 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.test.AbstractTest; import io.questdb.client.test.tools.TestUtils; -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * Tests for WebSocket transport support in the Sender.builder() API. * These tests verify the builder configuration and validation, @@ -650,7 +651,7 @@ public void testWsConfigString_uppercaseNotSupported() { public void testWsConfigString_withToken() throws Exception { assertMemoryLeak(() -> { int port = findUnusedPort(); - assertBadConfig("ws::addr=localhost:" + port + ";token=mytoken;", "connect", "Failed"); + assertBadConfig("ws::addr=localhost:" + port + ";token=mytoken;", "token is not supported"); }); } From 716f0a3a989f7869e91193f7e2bf3533cc23a5da Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 13:30:22 +0100 Subject: [PATCH 161/230] allow mid-row schema changes --- .../cutlass/qwp/client/QwpUdpSender.java | 100 +++++++----- .../cutlass/qwp/protocol/QwpTableBuffer.java | 37 +++++ .../cutlass/qwp/client/QwpUdpSenderTest.java | 154 +++++++++++++++++- 3 files changed, 241 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index cbf8c86..8e0395c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -99,6 +99,7 @@ public class QwpUdpSender implements Sender { private int estimateColumnCount; private QwpTableBuffer.ColumnBuffer[] missingColumns = new QwpTableBuffer.ColumnBuffer[8]; private int missingColumnCount; + private boolean replayingRowJournal; private int rowJournalSize; private long runningEstimate; @@ -158,6 +159,7 @@ public void cancelRow() { checkNotClosed(); if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); + currentTableBuffer.rollbackUncommittedColumns(); rollbackEstimateToCommitted(); } rowJournalSize = 0; @@ -169,6 +171,7 @@ public void close() { try { if (hasInProgressRow()) { currentTableBuffer.cancelCurrentRow(); + currentTableBuffer.rollbackUncommittedColumns(); rollbackEstimateToCommitted(); rowJournalSize = 0; } @@ -419,12 +422,13 @@ public Sender timestampColumn(CharSequence columnName, Instant value) { private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean nullable) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn(name, type); - if (col == null && currentTableBuffer.getRowCount() > 0) { + assert !replayingRowJournal || currentTableBuffer.getRowCount() == 0; + if (col == null && !replayingRowJournal && currentTableBuffer.getRowCount() > 0) { if (currentRowColumnCount > 0) { - throw new LineSenderException("schema change in middle of row is not supported"); + flushCommittedRowsAndReplayCurrentRow(); + } else { + flushSingleTable(currentTableName, currentTableBuffer); } - flushSingleTable(currentTableName, currentTableBuffer); - col = currentTableBuffer.getOrCreateColumn(name, type, nullable); } if (col == null) { @@ -444,7 +448,7 @@ private void appendBooleanColumn(CharSequence name, boolean value, boolean addJo applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_BOOL; e.name = col.getName(); @@ -462,7 +466,7 @@ private void appendDecimal128Column(CharSequence name, Decimal128 value, boolean applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL128; e.name = col.getName(); @@ -480,7 +484,7 @@ private void appendDecimal256Column(CharSequence name, Decimal256 value, boolean applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL256; e.name = col.getName(); @@ -498,7 +502,7 @@ private void appendDecimal64Column(CharSequence name, Decimal64 value, boolean a applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL64; e.name = col.getName(); @@ -531,7 +535,7 @@ private void appendDesignatedTimestamp(long value, boolean nanos, boolean addJou long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS; e.longValue = value; @@ -567,7 +571,7 @@ private void appendDoubleArrayColumn(CharSequence name, Object value, boolean ad applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE_ARRAY; e.name = col.getName(); @@ -585,7 +589,7 @@ private void appendDoubleColumn(CharSequence name, double value, boolean addJour applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE; e.name = col.getName(); @@ -622,7 +626,7 @@ private void appendLongArrayColumn(CharSequence name, Object value, boolean addJ applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG_ARRAY; e.name = col.getName(); @@ -640,7 +644,7 @@ private void appendLongColumn(CharSequence name, long value, boolean addJournal) applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG; e.name = col.getName(); @@ -660,7 +664,7 @@ private void appendStringColumn(CharSequence name, CharSequence value, boolean a applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_STRING; e.name = col.getName(); @@ -679,7 +683,7 @@ private void appendSymbolColumn(CharSequence name, CharSequence value, boolean a applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_SYMBOL; e.name = col.getName(); @@ -697,7 +701,7 @@ private void appendTimestampColumn(CharSequence name, byte type, long value, byt applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; - if (addJournal && trackDatagramEstimate) { + if (shouldJournal(addJournal)) { ColumnEntry e = nextJournalEntry(); e.kind = journalKind; e.name = col.getName(); @@ -927,6 +931,14 @@ private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { resetEstimateState(); } + private void flushCommittedRowsAndReplayCurrentRow() { + currentTableBuffer.cancelCurrentRow(); + currentTableBuffer.rollbackUncommittedColumns(); + rollbackEstimateToCommitted(); + flushSingleTable(currentTableName, currentTableBuffer); + replayRowJournal(); + } + private void maybeAutoFlush() { if (runningEstimate <= maxDatagramSize) { return; @@ -936,11 +948,7 @@ private void maybeAutoFlush() { throw singleRowTooLarge(runningEstimate); } - currentTableBuffer.cancelCurrentRow(); - rollbackEstimateToCommitted(); - - flushSingleTable(currentTableName, currentTableBuffer); - replayRowJournal(); + flushCommittedRowsAndReplayCurrentRow(); collectMissingColumns(currentTableBuffer.getRowCount() + 1); if (runningEstimate > maxDatagramSize) { @@ -986,27 +994,33 @@ private ColumnEntry nextJournalEntry() { } private void replayRowJournal() { - for (int i = 0; i < rowJournalSize; i++) { - ColumnEntry entry = rowJournal.getQuick(i); - switch (entry.kind) { - case ENTRY_AT_MICROS -> appendDesignatedTimestamp(entry.longValue, false, false); - case ENTRY_AT_NANOS -> appendDesignatedTimestamp(entry.longValue, true, false); - case ENTRY_BOOL -> appendBooleanColumn(entry.name, entry.boolValue, false); - case ENTRY_DECIMAL128 -> appendDecimal128Column(entry.name, (Decimal128) entry.objectValue, false); - case ENTRY_DECIMAL256 -> appendDecimal256Column(entry.name, (Decimal256) entry.objectValue, false); - case ENTRY_DECIMAL64 -> appendDecimal64Column(entry.name, (Decimal64) entry.objectValue, false); - case ENTRY_DOUBLE -> appendDoubleColumn(entry.name, entry.doubleValue, false); - case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayColumn(entry.name, entry.objectValue, false); - case ENTRY_LONG -> appendLongColumn(entry.name, entry.longValue, false); - case ENTRY_LONG_ARRAY -> appendLongArrayColumn(entry.name, entry.objectValue, false); - case ENTRY_STRING -> appendStringColumn(entry.name, entry.stringValue, false); - case ENTRY_SYMBOL -> appendSymbolColumn(entry.name, entry.stringValue, false); - case ENTRY_TIMESTAMP_COL_MICROS -> - appendTimestampColumn(entry.name, TYPE_TIMESTAMP, entry.longValue, ENTRY_TIMESTAMP_COL_MICROS, false); - case ENTRY_TIMESTAMP_COL_NANOS -> - appendTimestampColumn(entry.name, TYPE_TIMESTAMP_NANOS, entry.longValue, ENTRY_TIMESTAMP_COL_NANOS, false); - default -> throw new LineSenderException("unknown row journal entry type: " + entry.kind); + assert currentTableBuffer.getRowCount() == 0 : "row journal replay requires a reset table buffer"; + replayingRowJournal = true; + try { + for (int i = 0; i < rowJournalSize; i++) { + ColumnEntry entry = rowJournal.getQuick(i); + switch (entry.kind) { + case ENTRY_AT_MICROS -> appendDesignatedTimestamp(entry.longValue, false, false); + case ENTRY_AT_NANOS -> appendDesignatedTimestamp(entry.longValue, true, false); + case ENTRY_BOOL -> appendBooleanColumn(entry.name, entry.boolValue, false); + case ENTRY_DECIMAL128 -> appendDecimal128Column(entry.name, (Decimal128) entry.objectValue, false); + case ENTRY_DECIMAL256 -> appendDecimal256Column(entry.name, (Decimal256) entry.objectValue, false); + case ENTRY_DECIMAL64 -> appendDecimal64Column(entry.name, (Decimal64) entry.objectValue, false); + case ENTRY_DOUBLE -> appendDoubleColumn(entry.name, entry.doubleValue, false); + case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayColumn(entry.name, entry.objectValue, false); + case ENTRY_LONG -> appendLongColumn(entry.name, entry.longValue, false); + case ENTRY_LONG_ARRAY -> appendLongArrayColumn(entry.name, entry.objectValue, false); + case ENTRY_STRING -> appendStringColumn(entry.name, entry.stringValue, false); + case ENTRY_SYMBOL -> appendSymbolColumn(entry.name, entry.stringValue, false); + case ENTRY_TIMESTAMP_COL_MICROS -> + appendTimestampColumn(entry.name, TYPE_TIMESTAMP, entry.longValue, ENTRY_TIMESTAMP_COL_MICROS, false); + case ENTRY_TIMESTAMP_COL_NANOS -> + appendTimestampColumn(entry.name, TYPE_TIMESTAMP_NANOS, entry.longValue, ENTRY_TIMESTAMP_COL_NANOS, false); + default -> throw new LineSenderException("unknown row journal entry type: " + entry.kind); + } } + } finally { + replayingRowJournal = false; } } @@ -1030,6 +1044,10 @@ private void rollbackEstimateToCommitted() { missingColumnCount = 0; } + private boolean shouldJournal(boolean addJournal) { + return addJournal && (trackDatagramEstimate || currentTableBuffer.getRowCount() > 0 || rowJournalSize > 0); + } + private LineSenderException singleRowTooLarge(long estimate) { return new LineSenderException( "single row exceeds maximum datagram size (" + maxDatagramSize diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index b6c2128..0f0a9e5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -209,6 +209,43 @@ private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable return col; } + public void rollbackUncommittedColumns() { + if (columns.size() <= committedColumnCount) { + return; + } + + for (int i = columns.size() - 1; i >= committedColumnCount; i--) { + ColumnBuffer col = columns.getQuick(i); + if (col != null) { + col.close(); + } + columns.remove(i); + } + rebuildColumnAccessStructures(); + } + + private void rebuildColumnAccessStructures() { + columnNameToIndex.clear(); + + int columnCount = columns.size(); + int minCapacity = Math.max(8, columnCount + 4); + if (fastColumns == null || fastColumns.length < minCapacity) { + fastColumns = new ColumnBuffer[minCapacity]; + } else { + Arrays.fill(fastColumns, null); + } + + for (int i = 0; i < columnCount; i++) { + ColumnBuffer col = columns.getQuick(i); + fastColumns[i] = col; + columnNameToIndex.put(col.name, i); + } + + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; + } + private ColumnBuffer lookupColumn(CharSequence name, byte type) { // Fast path: predict next column in sequence int n = columns.size(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 12c9492..17fa2b2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -368,7 +368,7 @@ public void testBoundedSenderOutOfOrderExistingColumnsPreservesRowsAndPacketLimi } @Test - public void testMixingAtNowAndAtMicrosAfterCommittedRowsThrowsSchemaChange() throws Exception { + public void testMixingAtNowAndAtMicrosAfterCommittedRowsSplitsDatagramAndPreservesRows() throws Exception { assertMemoryLeak(() -> { CapturingNetworkFacade nf = new CapturingNetworkFacade(); try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { @@ -377,10 +377,17 @@ public void testMixingAtNowAndAtMicrosAfterCommittedRowsThrowsSchemaChange() thr .atNow(); sender.longColumn("x", 2); - assertThrowsContains("schema change in middle of row is not supported", - () -> sender.at(2, ChronoUnit.MICROS)); - sender.cancelRow(); + sender.at(2, ChronoUnit.MICROS); + Assert.assertEquals(1, nf.sendCount); + + sender.flush(); } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "x", 1L), + decodedRow("t", "x", 2L, "", 2L) + ), decodeRows(nf.packets)); }); } @@ -543,7 +550,7 @@ public void testOversizedArrayRowRejectedUsesActualEncodedSize() throws Exceptio } @Test - public void testSchemaChangeMidRowThrows() throws Exception { + public void testSchemaChangeMidRowFlushesImmediatelyAndPreservesRows() throws Exception { assertMemoryLeak(() -> { CapturingNetworkFacade nf = new CapturingNetworkFacade(); try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { @@ -552,14 +559,143 @@ public void testSchemaChangeMidRowThrows() throws Exception { .atNow(); sender.longColumn("a", 2); - assertThrowsContains("schema change in middle of row is not supported", () -> sender.longColumn("b", 3)); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + Assert.assertEquals(1, nf.sendCount); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 2L, "b", 3L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplay() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.stringColumn("c", "x"); + sender.longColumn("d", 4); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 2L, "b", 3L, "c", "x", "d", 4L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplayWithoutDatagramTracking() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); + + sender.stringColumn("c", "x"); + sender.longColumn("d", 4); + Assert.assertEquals(1, nf.sendCount); + + sender.atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 2L, "b", 3L, "c", "x", "d", 4L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testCancelRowAfterMidRowSchemaChangeDoesNotLeakSchema() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + sender.longColumn("b", 3); + Assert.assertEquals(1, nf.sendCount); sender.cancelRow(); - Assert.assertEquals(0, nf.sendCount); + sender.longColumn("a", 4).atNow(); + sender.flush(); } - Assert.assertEquals(1, nf.sendCount); - assertRowsEqual(Arrays.asList(decodedRow("t", "a", 1L)), decodeRows(nf.packets)); + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 4L) + ), decodeRows(nf.packets)); + }); + } + + @Test + public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() throws Exception { + assertMemoryLeak(() -> { + String large = repeat('x', 5000); + List oversizedRow = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("a", 2) + .stringColumn("s", large) + .atNow(), + "a", 2L, + "s", large) + ); + int maxDatagramSize = fullPacketSize(oversizedRow) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("a", 1) + .atNow(); + + sender.longColumn("a", 2); + assertThrowsContains("single row exceeds maximum datagram size", () -> { + sender.stringColumn("s", large); + sender.atNow(); + }); + Assert.assertEquals(1, nf.sendCount); + + sender.cancelRow(); + sender.longColumn("a", 3).atNow(); + sender.flush(); + } + + Assert.assertEquals(2, nf.sendCount); + assertRowsEqual(Arrays.asList( + decodedRow("t", "a", 1L), + decodedRow("t", "a", 3L) + ), decodeRows(nf.packets)); }); } From 05c2595857af5ec120e9d56b620941f6142dd55b Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 14:33:30 +0100 Subject: [PATCH 162/230] wip: less copying --- core/src/main/c/share/net.c | 35 +- core/src/main/c/share/net.h | 8 + core/src/main/c/windows/net.c | 44 +- .../cutlass/line/udp/UdpLineChannel.java | 6 + .../cutlass/qwp/client/NativeSegmentList.java | 119 +++ .../cutlass/qwp/client/QwpUdpSender.java | 713 +++++++++--------- .../client/SegmentedNativeBufferWriter.java | 183 +++++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 4 + .../java/io/questdb/client/network/Net.java | 2 + .../questdb/client/network/NetworkFacade.java | 4 +- .../client/network/NetworkFacadeImpl.java | 7 +- .../cutlass/qwp/client/QwpUdpSenderTest.java | 51 ++ 12 files changed, 806 insertions(+), 370 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/SegmentedNativeBufferWriter.java diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index c484149..24bebee 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include "net.h" #include @@ -356,4 +357,36 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendTo (JNIEnv *e, jclass cl, jint fd, jlong ptr, jint len, jlong sockaddr) { return (jint) sendto((int) fd, (const void *) ptr, (size_t) len, 0, (const struct sockaddr *) sockaddr, sizeof(struct sockaddr_in)); -} \ No newline at end of file +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendToScatter + (JNIEnv *e, jclass cl, jint fd, jlong segmentsPtr, jint segmentCount, jlong sockaddr) { + if (segmentCount <= 0) { + return 0; + } + + struct iovec *iov = calloc((size_t) segmentCount, sizeof(struct iovec)); + if (iov == NULL) { + errno = ENOMEM; + return -1; + } + + const char *segment = (const char *) segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + iov[i].iov_base = (void *) (uintptr_t) (*(const jlong *) segment); + iov[i].iov_len = (size_t) (*(const jlong *) (segment + 8)); + segment += 16; + } + + struct msghdr msg; + memset(&msg, 0, sizeof(msg)); + msg.msg_name = (void *) sockaddr; + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_iov = iov; + msg.msg_iovlen = (size_t) segmentCount; + + ssize_t sent; + RESTARTABLE(sendmsg((int) fd, &msg, 0), sent); + free(iov); + return sent < 0 ? -1 : (jint) sent; +} diff --git a/core/src/main/c/share/net.h b/core/src/main/c/share/net.h index c3f1016..13adafc 100644 --- a/core/src/main/c/share/net.h +++ b/core/src/main/c/share/net.h @@ -182,6 +182,14 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_setMulticastTtl JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendTo (JNIEnv *, jclass, jint, jlong, jint, jlong); +/* + * Class: io_questdb_client_network_Net + * Method: sendToScatter + * Signature: (IJIJ)I + */ +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendToScatter + (JNIEnv *, jclass, jint, jlong, jint, jlong); + #ifdef __cplusplus } #endif diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index 4475adb..ba04b98 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "../share/net.h" #include "errno.h" @@ -287,4 +288,45 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendTo SaveLastError(); } return result; -} \ No newline at end of file +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_sendToScatter + (JNIEnv *e, jclass cl, jint fd, jlong segmentsPtr, jint segmentCount, jlong sockaddr) { + if (segmentCount <= 0) { + return 0; + } + + WSABUF *buffers = calloc((size_t) segmentCount, sizeof(WSABUF)); + if (buffers == NULL) { + WSASetLastError(WSA_NOT_ENOUGH_MEMORY); + SaveLastError(); + return -1; + } + + const char *segment = (const char *) segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + buffers[i].buf = (CHAR *) (uintptr_t) (*(const jlong *) segment); + buffers[i].len = (ULONG) (*(const jlong *) (segment + 8)); + segment += 16; + } + + DWORD bytesSent = 0; + int result = WSASendTo( + (SOCKET) fd, + buffers, + (DWORD) segmentCount, + &bytesSent, + 0, + (const struct sockaddr *) sockaddr, + sizeof(struct sockaddr_in), + NULL, + NULL + ); + free(buffers); + + if (result == SOCKET_ERROR) { + SaveLastError(); + return -1; + } + return (jint) bytesSent; +} diff --git a/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java b/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java index 1b8c68e..babc249 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/udp/UdpLineChannel.java @@ -85,4 +85,10 @@ public void send(long ptr, int len) { throw new LineSenderException("send error").errno(nf.errno()); } } + + public void sendSegments(long segmentsPtr, int segmentCount, int totalLen) { + if (nf.sendToRawScatter(fd, segmentsPtr, segmentCount, sockaddr) != totalLen) { + throw new LineSenderException("send error").errno(nf.errno()); + } + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java new file mode 100644 index 0000000..f30bc4c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java @@ -0,0 +1,119 @@ +/* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +final class NativeSegmentList implements QuietCloseable { + static final int SEGMENT_SIZE = 16; + + private int capacity; + private long ptr; + private int size; + private long totalLength; + + NativeSegmentList() { + this(16); + } + + NativeSegmentList(int initialCapacity) { + this.capacity = Math.max(initialCapacity, 4); + this.ptr = Unsafe.malloc((long) capacity * SEGMENT_SIZE, MemoryTag.NATIVE_DEFAULT); + } + + void add(long address, long length) { + if (length <= 0) { + return; + } + ensureCapacity(size + 1); + long segmentPtr = ptr + (long) size * SEGMENT_SIZE; + Unsafe.getUnsafe().putLong(segmentPtr, address); + Unsafe.getUnsafe().putLong(segmentPtr + 8, length); + size++; + totalLength += length; + } + + void appendFrom(NativeSegmentList other) { + if (other.size == 0) { + return; + } + ensureCapacity(size + other.size); + Unsafe.getUnsafe().copyMemory( + other.ptr, + ptr + (long) size * SEGMENT_SIZE, + (long) other.size * SEGMENT_SIZE + ); + size += other.size; + totalLength += other.totalLength; + } + + long getAddress() { + return ptr; + } + + int getSegmentCount() { + return size; + } + + long getTotalLength() { + return totalLength; + } + + @Override + public void close() { + if (ptr != 0) { + Unsafe.free(ptr, (long) capacity * SEGMENT_SIZE, MemoryTag.NATIVE_DEFAULT); + ptr = 0; + capacity = 0; + size = 0; + totalLength = 0; + } + } + + void reset() { + size = 0; + totalLength = 0; + } + + private void ensureCapacity(int required) { + if (required <= capacity) { + return; + } + + int newCapacity = capacity; + while (newCapacity < required) { + newCapacity *= 2; + } + + ptr = Unsafe.realloc( + ptr, + (long) capacity * SEGMENT_SIZE, + (long) newCapacity * SEGMENT_SIZE, + MemoryTag.NATIVE_DEFAULT + ); + capacity = newCapacity; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 8e0395c..3060429 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -57,8 +57,9 @@ * symbol dictionaries (no global/delta dict) and full schema (no schema refs). *

    * When {@code maxDatagramSize > 0}, the sender automatically flushes before - * a datagram exceeds the size limit. The current in-progress row is cancelled, - * committed rows are flushed, and the in-progress row is replayed from a journal. + * a datagram exceeds the size limit. The in-progress row stays staged in sender + * state until commit, so committed table data can be flushed without replaying + * the row back into column storage. */ public class QwpUdpSender implements Sender { private static final byte ENTRY_AT_MICROS = 1; @@ -80,28 +81,28 @@ public class QwpUdpSender implements Sender { private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); private final ArraySizeCounter arraySizeCounter = new ArraySizeCounter(); - private final NativeBufferWriter buffer = new NativeBufferWriter(); private final UdpLineChannel channel; private final QwpColumnWriter columnWriter = new QwpColumnWriter(); + private final NativeBufferWriter headerBuffer = new NativeBufferWriter(); private final int maxDatagramSize; + private final SegmentedNativeBufferWriter payloadBuffer = new SegmentedNativeBufferWriter(); private final boolean trackDatagramEstimate; private final ObjList rowJournal = new ObjList<>(); + private final NativeSegmentList sendSegments = new NativeSegmentList(); private final CharSequenceObjHashMap tableBuffers; + private QwpTableBuffer.ColumnBuffer[] touchedColumns = new QwpTableBuffer.ColumnBuffer[8]; private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; private boolean closed; private long committedEstimate; - private int committedEstimateColumnCount; private int currentRowColumnCount; private QwpTableBuffer currentTableBuffer; private String currentTableName; - private int estimateColumnCount; private QwpTableBuffer.ColumnBuffer[] missingColumns = new QwpTableBuffer.ColumnBuffer[8]; private int missingColumnCount; - private boolean replayingRowJournal; private int rowJournalSize; - private long runningEstimate; + private int touchedColumnCount; public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { this(nf, interfaceIPv4, sendToAddress, port, ttl, 0); @@ -160,9 +161,10 @@ public void cancelRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); currentTableBuffer.rollbackUncommittedColumns(); - rollbackEstimateToCommitted(); } - rowJournalSize = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + clearStagedRow(); } @Override @@ -172,8 +174,9 @@ public void close() { if (hasInProgressRow()) { currentTableBuffer.cancelCurrentRow(); currentTableBuffer.rollbackUncommittedColumns(); - rollbackEstimateToCommitted(); - rowJournalSize = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + clearStagedRow(); } flushInternal(); } catch (Exception e) { @@ -192,7 +195,9 @@ public void close() { } tableBuffers.clear(); channel.close(); - buffer.close(); + payloadBuffer.close(); + sendSegments.close(); + headerBuffer.close(); } } @@ -347,6 +352,7 @@ public void reset() { for (int i = 0, n = keys.size(); i < n; i++) { QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); if (buf != null) { + buf.rollbackUncommittedColumns(); buf.reset(); } } @@ -354,8 +360,8 @@ public void reset() { currentTableName = null; cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - rowJournalSize = 0; - resetEstimateState(); + clearStagedRow(); + resetCommittedEstimate(); } @Override @@ -386,8 +392,8 @@ public Sender table(CharSequence tableName) { } cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - rowJournalSize = 0; - resetEstimateState(); + clearStagedRow(); + resetCommittedEstimate(); currentTableName = tableName.toString(); currentTableBuffer = tableBuffers.get(currentTableName); @@ -422,90 +428,64 @@ public Sender timestampColumn(CharSequence columnName, Instant value) { private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean nullable) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn(name, type); - assert !replayingRowJournal || currentTableBuffer.getRowCount() == 0; - if (col == null && !replayingRowJournal && currentTableBuffer.getRowCount() > 0) { - if (currentRowColumnCount > 0) { - flushCommittedRowsAndReplayCurrentRow(); - } else { - flushSingleTable(currentTableName, currentTableBuffer); - } + if (col == null && currentTableBuffer.getRowCount() > 0) { + flushCommittedRowsOfCurrentTable(); + col = currentTableBuffer.getExistingColumn(name, type); } - if (col == null) { col = currentTableBuffer.getOrCreateColumn(name, type, nullable); } - syncSchemaEstimate(); return col; } private void appendBooleanColumn(CharSequence name, boolean value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addBoolean(value); - - long payloadDelta = packedBytes(col.getValueCount()) - packedBytes(valueCountBefore); - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_BOOL; - e.name = col.getName(); + e.column = col; e.boolValue = value; } } private void appendDecimal128Column(CharSequence name, Decimal128 value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addDecimal128(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 16; - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL128; - e.name = col.getName(); + e.column = col; e.objectValue = value; } } private void appendDecimal256Column(CharSequence name, Decimal256 value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addDecimal256(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 32; - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL256; - e.name = col.getName(); + e.column = col; e.objectValue = value; } } private void appendDecimal64Column(CharSequence name, Decimal64 value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addDecimal64(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DECIMAL64; - e.name = col.getName(); + e.column = col; e.objectValue = value; } } @@ -515,237 +495,115 @@ private void appendDesignatedTimestamp(long value, boolean nanos, boolean addJou if (nanos) { if (cachedTimestampNanosColumn == null) { cachedTimestampNanosColumn = acquireColumn("", TYPE_TIMESTAMP_NANOS, true); - } else { - syncSchemaEstimate(); } col = cachedTimestampNanosColumn; } else { if (cachedTimestampColumn == null) { cachedTimestampColumn = acquireColumn("", TYPE_TIMESTAMP, true); - } else { - syncSchemaEstimate(); } col = cachedTimestampColumn; } + addTouchedColumn(col); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addLong(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); - - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS; + e.column = col; e.longValue = value; } } private void appendDoubleArrayColumn(CharSequence name, Object value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); - int sizeBefore = col.getSize(); - - long payloadDelta; - if (value instanceof double[] values) { - payloadDelta = estimateArrayValueSize(1, values.length); - col.addDoubleArray(values); - } else if (value instanceof double[][] values) { - int dim0 = values.length; - int dim1 = dim0 > 0 ? values[0].length : 0; - payloadDelta = estimateArrayValueSize(2, (long) dim0 * dim1); - col.addDoubleArray(values); - } else if (value instanceof double[][][] values) { - int dim0 = values.length; - int dim1 = dim0 > 0 ? values[0].length : 0; - int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; - payloadDelta = estimateArrayValueSize(3, (long) dim0 * dim1 * dim2); - col.addDoubleArray(values); - } else if (value instanceof DoubleArray values) { - payloadDelta = estimateArrayValueSize(values); - col.addDoubleArray(values); - } else { - throw new LineSenderException("unsupported double array type"); - } - - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE_ARRAY; - e.name = col.getName(); + e.column = col; e.objectValue = value; } } private void appendDoubleColumn(CharSequence name, double value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addDouble(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_DOUBLE; - e.name = col.getName(); + e.column = col; e.doubleValue = value; } } private void appendLongArrayColumn(CharSequence name, Object value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); - int sizeBefore = col.getSize(); - - long payloadDelta; - if (value instanceof long[] values) { - payloadDelta = estimateArrayValueSize(1, values.length); - col.addLongArray(values); - } else if (value instanceof long[][] values) { - int dim0 = values.length; - int dim1 = dim0 > 0 ? values[0].length : 0; - payloadDelta = estimateArrayValueSize(2, (long) dim0 * dim1); - col.addLongArray(values); - } else if (value instanceof long[][][] values) { - int dim0 = values.length; - int dim1 = dim0 > 0 ? values[0].length : 0; - int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; - payloadDelta = estimateArrayValueSize(3, (long) dim0 * dim1 * dim2); - col.addLongArray(values); - } else if (value instanceof LongArray values) { - payloadDelta = estimateArrayValueSize(values); - col.addLongArray(values); - } else { - throw new LineSenderException("unsupported long array type"); - } - - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG_ARRAY; - e.name = col.getName(); + e.column = col; e.objectValue = value; } } private void appendLongColumn(CharSequence name, long value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addLong(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_LONG; - e.name = col.getName(); + e.column = col; e.longValue = value; } } private void appendStringColumn(CharSequence name, CharSequence value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - long stringBytesBefore = col.getStringDataSize(); - col.addString(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 4 - + (col.getStringDataSize() - stringBytesBefore); - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_STRING; - e.name = col.getName(); + e.column = col; e.stringValue = Chars.toString(value); } } private void appendSymbolColumn(CharSequence name, CharSequence value, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - int dictSizeBefore = col.getSymbolDictionarySize(); - col.addSymbol(value); - - long payloadDelta = estimateSymbolPayloadDelta(col, valueCountBefore, dictSizeBefore, value); - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = ENTRY_SYMBOL; - e.name = col.getName(); + e.column = col; e.stringValue = Chars.toString(value); } } private void appendTimestampColumn(CharSequence name, byte type, long value, byte journalKind, boolean addJournal) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); - int sizeBefore = col.getSize(); - int valueCountBefore = col.getValueCount(); - col.addLong(value); - - long payloadDelta = (long) (col.getValueCount() - valueCountBefore) * 8; - applyValueEstimate(col, sizeBefore, col.getSize(), payloadDelta); currentRowColumnCount++; + addTouchedColumn(col); - if (shouldJournal(addJournal)) { + if (addJournal) { ColumnEntry e = nextJournalEntry(); e.kind = journalKind; - e.name = col.getName(); + e.column = col; e.longValue = value; } } - private void collectMissingColumns(int targetRows) { - missingColumnCount = 0; - for (int i = 0, n = currentTableBuffer.getColumnCount(); i < n; i++) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); - int sizeBefore = col.getSize(); - int missing = targetRows - sizeBefore; - if (missing <= 0) { - continue; - } - - ensureMissingColumnCapacity(missingColumnCount + 1); - missingColumns[missingColumnCount++] = col; - - if (!trackDatagramEstimate) { - continue; - } - - if (col.isNullable()) { - runningEstimate += bitmapBytes(sizeBefore + missing) - bitmapBytes(sizeBefore); - continue; - } - - int valuesBefore = col.getValueCount(); - runningEstimate += nonNullablePaddingCost(col.getType(), valuesBefore, missing); - } - } - - private void applyValueEstimate(QwpTableBuffer.ColumnBuffer col, int sizeBefore, int sizeAfter, long payloadDelta) { - if (!trackDatagramEstimate) { - return; - } - runningEstimate += payloadDelta; - if (col.isNullable()) { - runningEstimate += bitmapBytes(sizeAfter) - bitmapBytes(sizeBefore); - } - } - private void atMicros(long timestampMicros) { if (currentRowColumnCount == 0) { throw new LineSenderException("no columns were provided"); @@ -774,26 +632,63 @@ private void checkTableSelected() { } } + private void clearStagedRow() { + for (int i = 0; i < rowJournalSize; i++) { + ColumnEntry entry = rowJournal.getQuick(i); + entry.column = null; + entry.objectValue = null; + entry.stringValue = null; + } + rowJournalSize = 0; + currentRowColumnCount = 0; + touchedColumnCount = 0; + missingColumnCount = 0; + } + + private void collectMissingColumns(int targetRows) { + missingColumnCount = 0; + for (int i = 0, n = currentTableBuffer.getColumnCount(); i < n; i++) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); + if (isTouchedColumn(col)) { + continue; + } + if (col.getSize() >= targetRows) { + continue; + } + ensureMissingColumnCapacity(missingColumnCount + 1); + missingColumns[missingColumnCount++] = col; + } + } + private void commitCurrentRow() { if (currentRowColumnCount == 0) { throw new LineSenderException("no columns were provided"); } + long estimate = 0; int targetRows = currentTableBuffer.getRowCount() + 1; collectMissingColumns(targetRows); - if (trackDatagramEstimate) { - maybeAutoFlush(); + estimate = estimateCurrentDatagramSizeWithStagedRow(targetRows); + if (estimate > maxDatagramSize) { + if (currentTableBuffer.getRowCount() == 0) { + throw singleRowTooLarge(estimate); + } + flushCommittedRowsOfCurrentTable(); + targetRows = currentTableBuffer.getRowCount() + 1; + collectMissingColumns(targetRows); + estimate = estimateCurrentDatagramSizeWithStagedRow(targetRows); + if (estimate > maxDatagramSize) { + throw singleRowTooLarge(estimate); + } + } } - currentTableBuffer.nextRow(missingColumns, missingColumnCount); + materializeCurrentRow(); if (trackDatagramEstimate) { - committedEstimate = runningEstimate; - committedEstimateColumnCount = estimateColumnCount; + committedEstimate = estimate; } - currentRowColumnCount = 0; - missingColumnCount = 0; - rowJournalSize = 0; + clearStagedRow(); } private void ensureNoInProgressRow(String operation) { @@ -805,23 +700,12 @@ private void ensureNoInProgressRow(String operation) { } } - private int encodeForUdp(QwpTableBuffer tableBuffer) { - buffer.reset(); - // Write 12-byte QWP1 header: magic, version, flags=0, tableCount=1, payloadLength=0 (patched later) - buffer.putByte((byte) 'Q'); - buffer.putByte((byte) 'W'); - buffer.putByte((byte) 'P'); - buffer.putByte((byte) '1'); - buffer.putByte(VERSION_1); - buffer.putByte((byte) 0); // flags - buffer.putShort((short) 1); // tableCount - buffer.putInt(0); // payloadLength placeholder - int payloadStart = buffer.getPosition(); - columnWriter.setBuffer(buffer); + private int encodePayloadForUdp(QwpTableBuffer tableBuffer) { + payloadBuffer.reset(); + columnWriter.setBuffer(payloadBuffer); columnWriter.encodeTable(tableBuffer, false, false, false); - int payloadLength = buffer.getPosition() - payloadStart; - buffer.patchInt(8, payloadLength); - return buffer.getPosition(); + payloadBuffer.finish(); + return payloadBuffer.getPosition(); } private long estimateArrayValueSize(int nDims, long elementCount) { @@ -840,6 +724,114 @@ private long estimateArrayValueSize(LongArray array) { return arraySizeCounter.size; } + private long estimateBaseForCurrentSchema() { + long estimate = HEADER_SIZE; + int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName); + estimate += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; + estimate += VARINT_INT_UPPER_BOUND; + estimate += VARINT_INT_UPPER_BOUND; + estimate += 1; + + QwpColumnDef[] defs = currentTableBuffer.getColumnDefs(); + for (QwpColumnDef def : defs) { + int nameUtf8 = NativeBufferWriter.utf8Length(def.getName()); + estimate += NativeBufferWriter.varintSize(nameUtf8) + nameUtf8; + estimate += 1; + + byte type = def.getTypeCode(); + if (type == TYPE_STRING || type == TYPE_VARCHAR) { + estimate += 4; + } else if (type == TYPE_SYMBOL) { + estimate += 1; + } else if (type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128 || type == TYPE_DECIMAL256) { + estimate += 1; + } + } + estimate += SAFETY_MARGIN_BYTES; + return estimate; + } + + private long estimateCurrentDatagramSizeWithStagedRow(int targetRows) { + long estimate = currentTableBuffer.getRowCount() > 0 ? committedEstimate : estimateBaseForCurrentSchema(); + for (int i = 0; i < rowJournalSize; i++) { + ColumnEntry entry = rowJournal.getQuick(i); + QwpTableBuffer.ColumnBuffer col = entry.column; + estimate += estimateEntryPayload(entry, col); + if (col.isNullable()) { + estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); + } + } + for (int i = 0; i < missingColumnCount; i++) { + QwpTableBuffer.ColumnBuffer col = missingColumns[i]; + int missing = targetRows - col.getSize(); + if (col.isNullable()) { + estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); + } else { + estimate += nonNullablePaddingCost(col.getType(), col.getValueCount(), missing); + } + } + return estimate; + } + + private long estimateEntryPayload(ColumnEntry entry, QwpTableBuffer.ColumnBuffer col) { + int valueCountBefore = col.getValueCount(); + return switch (entry.kind) { + case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_DOUBLE, ENTRY_LONG, + ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> 8; + case ENTRY_BOOL -> packedBytes(valueCountBefore + 1) - packedBytes(valueCountBefore); + case ENTRY_DECIMAL64 -> 8; + case ENTRY_DECIMAL128 -> 16; + case ENTRY_DECIMAL256 -> 32; + case ENTRY_DOUBLE_ARRAY -> estimateDoubleArrayPayload(entry.objectValue); + case ENTRY_LONG_ARRAY -> estimateLongArrayPayload(entry.objectValue); + case ENTRY_STRING -> entry.stringValue == null ? 0 : 4L + utf8Length(entry.stringValue); + case ENTRY_SYMBOL -> estimateSymbolPayloadDelta(col, valueCountBefore, entry.stringValue); + default -> throw new LineSenderException("unknown staged row entry type: " + entry.kind); + }; + } + + private long estimateDoubleArrayPayload(Object value) { + if (value instanceof double[] values) { + return estimateArrayValueSize(1, values.length); + } + if (value instanceof double[][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + return estimateArrayValueSize(2, (long) dim0 * dim1); + } + if (value instanceof double[][][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + return estimateArrayValueSize(3, (long) dim0 * dim1 * dim2); + } + if (value instanceof DoubleArray values) { + return estimateArrayValueSize(values); + } + throw new LineSenderException("unsupported double array type"); + } + + private long estimateLongArrayPayload(Object value) { + if (value instanceof long[] values) { + return estimateArrayValueSize(1, values.length); + } + if (value instanceof long[][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + return estimateArrayValueSize(2, (long) dim0 * dim1); + } + if (value instanceof long[][][] values) { + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + return estimateArrayValueSize(3, (long) dim0 * dim1 * dim2); + } + if (value instanceof LongArray values) { + return estimateArrayValueSize(values); + } + throw new LineSenderException("unsupported long array type"); + } + private void ensureMissingColumnCapacity(int required) { if (required <= missingColumns.length) { return; @@ -855,43 +847,60 @@ private void ensureMissingColumnCapacity(int required) { missingColumns = newArr; } - private long estimateSymbolPayloadDelta( - QwpTableBuffer.ColumnBuffer col, - int valueCountBefore, - int dictSizeBefore, - CharSequence value - ) { - int valueCountAfter = col.getValueCount(); - if (valueCountAfter == valueCountBefore) { + private void ensureTouchedColumnCapacity(int required) { + if (required <= touchedColumns.length) { + return; + } + + int newCapacity = touchedColumns.length; + while (newCapacity < required) { + newCapacity *= 2; + } + + QwpTableBuffer.ColumnBuffer[] newArr = new QwpTableBuffer.ColumnBuffer[newCapacity]; + System.arraycopy(touchedColumns, 0, newArr, 0, touchedColumnCount); + touchedColumns = newArr; + } + + private long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, int valueCountBefore, CharSequence value) { + if (value == null) { return 0; } - int dictSizeAfter = col.getSymbolDictionarySize(); - if (dictSizeAfter == dictSizeBefore) { - int maxIndex = Math.max(0, dictSizeAfter - 1); + int dictSizeBefore = col.getSymbolDictionarySize(); + if (col.hasSymbol(value)) { + int maxIndex = Math.max(0, dictSizeBefore - 1); return NativeBufferWriter.varintSize(maxIndex); } + int dictSizeAfter = dictSizeBefore + 1; long delta = 0; int utf8Len = utf8Length(value); delta += NativeBufferWriter.varintSize(utf8Len) + utf8Len; - delta += NativeBufferWriter.varintSize(dictSizeAfter) - - NativeBufferWriter.varintSize(dictSizeBefore); + delta += NativeBufferWriter.varintSize(dictSizeAfter) - NativeBufferWriter.varintSize(dictSizeBefore); if (dictSizeBefore > 0 && valueCountBefore > 0) { int oldMax = dictSizeBefore - 1; int newMax = dictSizeAfter - 1; delta += (long) valueCountBefore * ( - NativeBufferWriter.varintSize(newMax) - - NativeBufferWriter.varintSize(oldMax) + NativeBufferWriter.varintSize(newMax) - NativeBufferWriter.varintSize(oldMax) ); } - int newMax = dictSizeAfter - 1; - delta += NativeBufferWriter.varintSize(newMax); + delta += NativeBufferWriter.varintSize(dictSizeAfter - 1); return delta; } + private void flushCommittedRowsOfCurrentTable() { + if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0) { + return; + } + sendTableBuffer(currentTableName, currentTableBuffer); + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + resetCommittedEstimate(); + } + private void flushInternal() { ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { @@ -903,56 +912,90 @@ private void flushInternal() { if (tableBuffer == null || tableBuffer.getRowCount() == 0) { continue; } - - int len = encodeForUdp(tableBuffer); - try { - channel.send(buffer.getBufferPtr(), len); - } catch (LineSenderException e) { - LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); - } - tableBuffer.reset(); + sendTableBuffer(tableName, tableBuffer); } cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - rowJournalSize = 0; - resetEstimateState(); + clearStagedRow(); + resetCommittedEstimate(); } private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { - int len = encodeForUdp(tableBuffer); - try { - channel.send(buffer.getBufferPtr(), len); - } catch (LineSenderException e) { - LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); - } - tableBuffer.reset(); + sendTableBuffer(tableName, tableBuffer); cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - resetEstimateState(); + clearStagedRow(); + resetCommittedEstimate(); } - private void flushCommittedRowsAndReplayCurrentRow() { - currentTableBuffer.cancelCurrentRow(); - currentTableBuffer.rollbackUncommittedColumns(); - rollbackEstimateToCommitted(); - flushSingleTable(currentTableName, currentTableBuffer); - replayRowJournal(); + private boolean hasInProgressRow() { + return rowJournalSize > 0; } - private void maybeAutoFlush() { - if (runningEstimate <= maxDatagramSize) { - return; + private boolean isTouchedColumn(QwpTableBuffer.ColumnBuffer col) { + for (int i = 0; i < touchedColumnCount; i++) { + if (touchedColumns[i] == col) { + return true; + } } + return false; + } - if (currentTableBuffer.getRowCount() == 0) { - throw singleRowTooLarge(runningEstimate); + private void addTouchedColumn(QwpTableBuffer.ColumnBuffer col) { + if (isTouchedColumn(col)) { + return; } + ensureTouchedColumnCapacity(touchedColumnCount + 1); + touchedColumns[touchedColumnCount++] = col; + } + + private void materializeCurrentRow() { + for (int i = 0; i < rowJournalSize; i++) { + ColumnEntry entry = rowJournal.getQuick(i); + QwpTableBuffer.ColumnBuffer col = entry.column; + switch (entry.kind) { + case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> + col.addLong(entry.longValue); + case ENTRY_BOOL -> col.addBoolean(entry.boolValue); + case ENTRY_DECIMAL64 -> col.addDecimal64((Decimal64) entry.objectValue); + case ENTRY_DECIMAL128 -> col.addDecimal128((Decimal128) entry.objectValue); + case ENTRY_DECIMAL256 -> col.addDecimal256((Decimal256) entry.objectValue); + case ENTRY_DOUBLE -> col.addDouble(entry.doubleValue); + case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayValue(col, entry.objectValue); + case ENTRY_LONG_ARRAY -> appendLongArrayValue(col, entry.objectValue); + case ENTRY_STRING -> col.addString(entry.stringValue); + case ENTRY_SYMBOL -> col.addSymbol(entry.stringValue); + default -> throw new LineSenderException("unknown staged row entry type: " + entry.kind); + } + } + currentTableBuffer.nextRow(missingColumns, missingColumnCount); + } - flushCommittedRowsAndReplayCurrentRow(); - collectMissingColumns(currentTableBuffer.getRowCount() + 1); + private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { + if (value instanceof double[] values) { + col.addDoubleArray(values); + } else if (value instanceof double[][] values) { + col.addDoubleArray(values); + } else if (value instanceof double[][][] values) { + col.addDoubleArray(values); + } else if (value instanceof DoubleArray values) { + col.addDoubleArray(values); + } else { + throw new LineSenderException("unsupported double array type"); + } + } - if (runningEstimate > maxDatagramSize) { - throw singleRowTooLarge(runningEstimate); + private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { + if (value instanceof long[] values) { + col.addLongArray(values); + } else if (value instanceof long[][] values) { + col.addLongArray(values); + } else if (value instanceof long[][][] values) { + col.addLongArray(values); + } else if (value instanceof LongArray values) { + col.addLongArray(values); + } else { + throw new LineSenderException("unsupported long array type"); } } @@ -982,6 +1025,7 @@ private static int packedBytes(int valueCount) { private ColumnEntry nextJournalEntry() { if (rowJournalSize < rowJournal.size()) { ColumnEntry entry = rowJournal.getQuick(rowJournalSize); + entry.column = null; entry.objectValue = null; entry.stringValue = null; rowJournalSize++; @@ -993,59 +1037,36 @@ private ColumnEntry nextJournalEntry() { return entry; } - private void replayRowJournal() { - assert currentTableBuffer.getRowCount() == 0 : "row journal replay requires a reset table buffer"; - replayingRowJournal = true; - try { - for (int i = 0; i < rowJournalSize; i++) { - ColumnEntry entry = rowJournal.getQuick(i); - switch (entry.kind) { - case ENTRY_AT_MICROS -> appendDesignatedTimestamp(entry.longValue, false, false); - case ENTRY_AT_NANOS -> appendDesignatedTimestamp(entry.longValue, true, false); - case ENTRY_BOOL -> appendBooleanColumn(entry.name, entry.boolValue, false); - case ENTRY_DECIMAL128 -> appendDecimal128Column(entry.name, (Decimal128) entry.objectValue, false); - case ENTRY_DECIMAL256 -> appendDecimal256Column(entry.name, (Decimal256) entry.objectValue, false); - case ENTRY_DECIMAL64 -> appendDecimal64Column(entry.name, (Decimal64) entry.objectValue, false); - case ENTRY_DOUBLE -> appendDoubleColumn(entry.name, entry.doubleValue, false); - case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayColumn(entry.name, entry.objectValue, false); - case ENTRY_LONG -> appendLongColumn(entry.name, entry.longValue, false); - case ENTRY_LONG_ARRAY -> appendLongArrayColumn(entry.name, entry.objectValue, false); - case ENTRY_STRING -> appendStringColumn(entry.name, entry.stringValue, false); - case ENTRY_SYMBOL -> appendSymbolColumn(entry.name, entry.stringValue, false); - case ENTRY_TIMESTAMP_COL_MICROS -> - appendTimestampColumn(entry.name, TYPE_TIMESTAMP, entry.longValue, ENTRY_TIMESTAMP_COL_MICROS, false); - case ENTRY_TIMESTAMP_COL_NANOS -> - appendTimestampColumn(entry.name, TYPE_TIMESTAMP_NANOS, entry.longValue, ENTRY_TIMESTAMP_COL_NANOS, false); - default -> throw new LineSenderException("unknown row journal entry type: " + entry.kind); - } - } - } finally { - replayingRowJournal = false; - } - } - - private void resetEstimateState() { - runningEstimate = 0; + private void resetCommittedEstimate() { committedEstimate = 0; - estimateColumnCount = 0; - committedEstimateColumnCount = 0; - currentRowColumnCount = 0; - missingColumnCount = 0; } - private boolean hasInProgressRow() { - return currentTableBuffer != null && currentTableBuffer.hasInProgressRow(); - } + private void sendTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer) { + int payloadLength = encodePayloadForUdp(tableBuffer); + headerBuffer.reset(); + headerBuffer.putByte((byte) 'Q'); + headerBuffer.putByte((byte) 'W'); + headerBuffer.putByte((byte) 'P'); + headerBuffer.putByte((byte) '1'); + headerBuffer.putByte(VERSION_1); + headerBuffer.putByte((byte) 0); + headerBuffer.putShort((short) 1); + headerBuffer.putInt(payloadLength); - private void rollbackEstimateToCommitted() { - runningEstimate = committedEstimate; - estimateColumnCount = committedEstimateColumnCount; - currentRowColumnCount = 0; - missingColumnCount = 0; - } + sendSegments.reset(); + sendSegments.add(headerBuffer.getBufferPtr(), headerBuffer.getPosition()); + sendSegments.appendFrom(payloadBuffer.getSegments()); - private boolean shouldJournal(boolean addJournal) { - return addJournal && (trackDatagramEstimate || currentTableBuffer.getRowCount() > 0 || rowJournalSize > 0); + try { + channel.sendSegments( + sendSegments.getAddress(), + sendSegments.getSegmentCount(), + (int) sendSegments.getTotalLength() + ); + } catch (LineSenderException e) { + LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); + } + tableBuffer.reset(); } private LineSenderException singleRowTooLarge(long estimate) { @@ -1055,46 +1076,6 @@ private LineSenderException singleRowTooLarge(long estimate) { ); } - private void syncSchemaEstimate() { - if (!trackDatagramEstimate) { - return; - } - - int newColumnCount = currentTableBuffer.getColumnCount(); - if (newColumnCount == estimateColumnCount) { - return; - } - - if (estimateColumnCount == 0) { - long base = HEADER_SIZE; - int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName); - base += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; - base += VARINT_INT_UPPER_BOUND; // row count varint upper bound - base += VARINT_INT_UPPER_BOUND; // column count varint upper bound - base += 1; // schema mode byte - base += SAFETY_MARGIN_BYTES; - runningEstimate += base; - } - - QwpColumnDef[] defs = currentTableBuffer.getColumnDefs(); - for (int i = estimateColumnCount; i < newColumnCount; i++) { - QwpColumnDef def = defs[i]; - int nameUtf8 = NativeBufferWriter.utf8Length(def.getName()); - runningEstimate += NativeBufferWriter.varintSize(nameUtf8) + nameUtf8; - runningEstimate += 1; // wire type code - - byte type = def.getTypeCode(); - if (type == TYPE_STRING || type == TYPE_VARCHAR) { - runningEstimate += 4; // offset[0] - } else if (type == TYPE_SYMBOL) { - runningEstimate += 1; // varintSize(0) for empty dictionary length - } else if (type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128 || type == TYPE_DECIMAL256) { - runningEstimate += 1; // scale byte - } - } - estimateColumnCount = newColumnCount; - } - private long toMicros(long value, ChronoUnit unit) { return switch (unit) { case NANOS -> value / 1000L; @@ -1170,10 +1151,10 @@ private void reset() { private static class ColumnEntry { boolean boolValue; + QwpTableBuffer.ColumnBuffer column; double doubleValue; byte kind; long longValue; - String name; Object objectValue; String stringValue; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/SegmentedNativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/SegmentedNativeBufferWriter.java new file mode 100644 index 0000000..f7ca788 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/SegmentedNativeBufferWriter.java @@ -0,0 +1,183 @@ +/* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; + +final class SegmentedNativeBufferWriter implements QwpBufferWriter, QuietCloseable { + private final ObjList chunks = new ObjList<>(); + private final NativeSegmentList segments = new NativeSegmentList(); + + private NativeBufferWriter currentChunk; + private long flushedBytes; + private int nextChunkIndex; + + SegmentedNativeBufferWriter() { + currentChunk = new NativeBufferWriter(); + chunks.add(currentChunk); + } + + @Override + public void close() { + for (int i = 0, n = chunks.size(); i < n; i++) { + chunks.getQuick(i).close(); + } + chunks.clear(); + segments.close(); + } + + void finish() { + flushCurrentChunk(); + } + + NativeSegmentList getSegments() { + return segments; + } + + @Override + public void ensureCapacity(int additionalBytes) { + currentChunk.ensureCapacity(additionalBytes); + } + + @Override + public long getBufferPtr() { + return currentChunk.getBufferPtr(); + } + + @Override + public int getCapacity() { + return currentChunk.getCapacity(); + } + + @Override + public int getPosition() { + return (int) (flushedBytes + currentChunk.getPosition()); + } + + @Override + public void patchInt(int offset, int value) { + if (offset < flushedBytes || offset + Integer.BYTES > flushedBytes + currentChunk.getPosition()) { + throw new UnsupportedOperationException("cannot patch flushed segment data"); + } + currentChunk.patchInt((int) (offset - flushedBytes), value); + } + + @Override + public void putBlockOfBytes(long from, long len) { + flushCurrentChunk(); + segments.add(from, len); + flushedBytes += len; + } + + @Override + public void putByte(byte value) { + currentChunk.putByte(value); + } + + @Override + public void putDouble(double value) { + currentChunk.putDouble(value); + } + + @Override + public void putFloat(float value) { + currentChunk.putFloat(value); + } + + @Override + public void putInt(int value) { + currentChunk.putInt(value); + } + + @Override + public void putLong(long value) { + currentChunk.putLong(value); + } + + @Override + public void putLongBE(long value) { + currentChunk.putLongBE(value); + } + + @Override + public void putShort(short value) { + currentChunk.putShort(value); + } + + @Override + public void putString(String value) { + currentChunk.putString(value); + } + + @Override + public void putUtf8(String value) { + currentChunk.putUtf8(value); + } + + @Override + public void putVarint(long value) { + currentChunk.putVarint(value); + } + + @Override + public void reset() { + segments.reset(); + flushedBytes = 0; + nextChunkIndex = 0; + for (int i = 0, n = chunks.size(); i < n; i++) { + chunks.getQuick(i).reset(); + } + currentChunk = chunks.getQuick(0); + } + + @Override + public void skip(int bytes) { + currentChunk.skip(bytes); + } + + private void flushCurrentChunk() { + int chunkSize = currentChunk.getPosition(); + if (chunkSize == 0) { + return; + } + + segments.add(currentChunk.getBufferPtr(), chunkSize); + flushedBytes += chunkSize; + currentChunk = nextChunk(); + } + + private NativeBufferWriter nextChunk() { + nextChunkIndex++; + if (nextChunkIndex < chunks.size()) { + NativeBufferWriter chunk = chunks.getQuick(nextChunkIndex); + chunk.reset(); + return chunk; + } + + NativeBufferWriter chunk = new NativeBufferWriter(); + chunks.add(chunk); + return chunk; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 0f0a9e5..2f72c14 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -1161,6 +1161,10 @@ public int getSymbolDictionarySize() { return symbolList != null ? symbolList.size() : 0; } + public boolean hasSymbol(CharSequence value) { + return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + } + public byte getType() { return type; } diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index b1c4721..78619fb 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -122,6 +122,8 @@ public static void init() { public static native int sendTo(int fd, long ptr, int len, long sockaddr); + public static native int sendToScatter(int fd, long segmentsPtr, int segmentCount, long sockaddr); + public static native int setKeepAlive0(int fd, int seconds); public static native int setMulticastInterface(int fd, int ipv4address); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacade.java b/core/src/main/java/io/questdb/client/network/NetworkFacade.java index f4909b3..858f4e2 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacade.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacade.java @@ -55,6 +55,8 @@ public interface NetworkFacade { int sendToRaw(int fd, long lo, int len, long socketAddress); + int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress); + int setMulticastInterface(int fd, int ipv4Address); int setMulticastTtl(int fd, int ttl); @@ -78,4 +80,4 @@ public interface NetworkFacade { * @return true if a disconnect happened, false otherwise */ boolean testConnection(int fd, long buffer, int bufferSize); -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java index fd4abc1..2eed1a3 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java @@ -102,6 +102,11 @@ public int sendToRaw(int fd, long ptr, int len, long socketAddress) { return Net.sendTo(fd, ptr, len, socketAddress); } + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + return Net.sendToScatter(fd, segmentsPtr, segmentCount, socketAddress); + } + @Override public int setMulticastInterface(int fd, int ipv4Address) { return Net.setMulticastInterface(fd, ipv4Address); @@ -149,4 +154,4 @@ public boolean testConnection(int fd, long buffer, int bufferSize) { final int nRead = Net.peek(fd, buffer, bufferSize); return nRead < 0; } -} \ No newline at end of file +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 17fa2b2..faefe4d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -409,6 +409,23 @@ public void testFlushWhileRowInProgressThrowsAndPreservesRow() throws Exception }); } + @Test + public void testSimpleLongRowUsesScatterSendPath() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + sender.table("t").longColumn("x", 42L).atNow(); + sender.flush(); + } + + Assert.assertEquals(1, nf.sendCount); + Assert.assertEquals(1, nf.scatterSendCount); + Assert.assertEquals(0, nf.rawSendCount); + Assert.assertTrue("expected multiple segments for header/schema/data", nf.segmentCounts.get(0) > 1); + assertRowsEqual(Arrays.asList(decodedRow("t", "x", 42L)), decodeRows(nf.packets)); + }); + } + @Test public void testSwitchTableWhileRowInProgressThrowsAndPreservesRows() throws Exception { assertMemoryLeak(() -> { @@ -1038,9 +1055,14 @@ private interface ThrowingRunnable { } private static final class CapturingNetworkFacade extends NetworkFacadeImpl { + private static final int SEGMENT_SIZE = 16; + private final List lengths = new ArrayList<>(); private final List packets = new ArrayList<>(); + private final List segmentCounts = new ArrayList<>(); + private int rawSendCount; private int sendCount; + private int scatterSendCount; @Override public int close(int fd) { @@ -1058,10 +1080,39 @@ public int sendToRaw(int fd, long lo, int len, long socketAddress) { Unsafe.getUnsafe().copyMemory(null, lo, packet, Unsafe.BYTE_OFFSET, len); packets.add(packet); lengths.add(len); + rawSendCount++; sendCount++; return len; } + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + int packetLength = 0; + long segmentPtr = segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + packetLength += (int) Unsafe.getUnsafe().getLong(segmentPtr + 8); + segmentPtr += SEGMENT_SIZE; + } + + byte[] packet = new byte[packetLength]; + int offset = 0; + segmentPtr = segmentsPtr; + for (int i = 0; i < segmentCount; i++) { + long dataAddress = Unsafe.getUnsafe().getLong(segmentPtr); + int dataLength = (int) Unsafe.getUnsafe().getLong(segmentPtr + 8); + Unsafe.getUnsafe().copyMemory(null, dataAddress, packet, Unsafe.BYTE_OFFSET + offset, dataLength); + offset += dataLength; + segmentPtr += SEGMENT_SIZE; + } + + packets.add(packet); + lengths.add(packetLength); + segmentCounts.add(segmentCount); + scatterSendCount++; + sendCount++; + return packetLength; + } + @Override public int setMulticastInterface(int fd, int ipv4Address) { return 0; From fa6223db44a5723d943f8e06a4f25d70ee8c82cb Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 14:55:28 +0100 Subject: [PATCH 163/230] wip: less copying, part 2 --- .../cutlass/qwp/client/QwpUdpSender.java | 446 +++++++++++------- .../qwp/protocol/OffHeapAppendMemory.java | 10 + .../cutlass/qwp/protocol/QwpTableBuffer.java | 24 + .../cutlass/qwp/client/QwpUdpSenderTest.java | 51 ++ 4 files changed, 349 insertions(+), 182 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 3060429..ffc6cf3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -30,6 +30,7 @@ import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.cutlass.line.udp.UdpLineChannel; +import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory; import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; @@ -38,6 +39,7 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; import io.questdb.client.std.bytes.DirectByteSlice; import io.questdb.client.network.NetworkFacade; import org.jetbrains.annotations.NotNull; @@ -86,8 +88,8 @@ public class QwpUdpSender implements Sender { private final NativeBufferWriter headerBuffer = new NativeBufferWriter(); private final int maxDatagramSize; private final SegmentedNativeBufferWriter payloadBuffer = new SegmentedNativeBufferWriter(); + private final NativeRowStaging stagedRow = new NativeRowStaging(); private final boolean trackDatagramEstimate; - private final ObjList rowJournal = new ObjList<>(); private final NativeSegmentList sendSegments = new NativeSegmentList(); private final CharSequenceObjHashMap tableBuffers; private QwpTableBuffer.ColumnBuffer[] touchedColumns = new QwpTableBuffer.ColumnBuffer[8]; @@ -101,7 +103,6 @@ public class QwpUdpSender implements Sender { private String currentTableName; private QwpTableBuffer.ColumnBuffer[] missingColumns = new QwpTableBuffer.ColumnBuffer[8]; private int missingColumnCount; - private int rowJournalSize; private int touchedColumnCount; public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { @@ -146,7 +147,7 @@ public void atNow() { public Sender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - appendBooleanColumn(columnName, value, true); + appendBooleanColumn(columnName, value); return this; } @@ -198,6 +199,7 @@ public void close() { payloadBuffer.close(); sendSegments.close(); headerBuffer.close(); + stagedRow.close(); } } @@ -208,7 +210,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { } checkNotClosed(); checkTableSelected(); - appendDecimal64Column(name, value, true); + appendDecimal64Column(name, value); return this; } @@ -219,7 +221,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { } checkNotClosed(); checkTableSelected(); - appendDecimal128Column(name, value, true); + appendDecimal128Column(name, value); return this; } @@ -230,7 +232,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { } checkNotClosed(); checkTableSelected(); - appendDecimal256Column(name, value, true); + appendDecimal256Column(name, value); return this; } @@ -241,7 +243,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, values, true); + appendDoubleArrayColumn(name, values); return this; } @@ -252,7 +254,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, values, true); + appendDoubleArrayColumn(name, values); return this; } @@ -263,7 +265,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, values, true); + appendDoubleArrayColumn(name, values); return this; } @@ -274,7 +276,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, array, true); + appendDoubleArrayColumn(name, array); return this; } @@ -282,7 +284,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public Sender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - appendDoubleColumn(columnName, value, true); + appendDoubleColumn(columnName, value); return this; } @@ -300,7 +302,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, values, true); + appendLongArrayColumn(name, values); return this; } @@ -311,7 +313,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, values, true); + appendLongArrayColumn(name, values); return this; } @@ -322,7 +324,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, values, true); + appendLongArrayColumn(name, values); return this; } @@ -333,7 +335,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, array, true); + appendLongArrayColumn(name, array); return this; } @@ -341,7 +343,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public Sender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - appendLongColumn(columnName, value, true); + appendLongColumn(columnName, value); return this; } @@ -368,7 +370,7 @@ public void reset() { public Sender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - appendStringColumn(columnName, value, true); + appendStringColumn(columnName, value); return this; } @@ -376,7 +378,7 @@ public Sender stringColumn(CharSequence columnName, CharSequence value) { public Sender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - appendSymbolColumn(columnName, value, true); + appendSymbolColumn(columnName, value); return this; } @@ -409,10 +411,10 @@ public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit un checkNotClosed(); checkTableSelected(); if (unit == ChronoUnit.NANOS) { - appendTimestampColumn(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS, true); + appendTimestampColumn(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS); } else { long micros = toMicros(value, unit); - appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS, true); + appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS); } return this; } @@ -422,7 +424,7 @@ public Sender timestampColumn(CharSequence columnName, Instant value) { checkNotClosed(); checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS, true); + appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS); return this; } @@ -438,59 +440,35 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, return col; } - private void appendBooleanColumn(CharSequence name, boolean value, boolean addJournal) { + private void appendBooleanColumn(CharSequence name, boolean value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_BOOL; - e.column = col; - e.boolValue = value; - } + stagedRow.appendBoolean(col, value); } - private void appendDecimal128Column(CharSequence name, Decimal128 value, boolean addJournal) { + private void appendDecimal128Column(CharSequence name, Decimal128 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DECIMAL128; - e.column = col; - e.objectValue = value; - } + stagedRow.appendDecimal128(col, value); } - private void appendDecimal256Column(CharSequence name, Decimal256 value, boolean addJournal) { + private void appendDecimal256Column(CharSequence name, Decimal256 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DECIMAL256; - e.column = col; - e.objectValue = value; - } + stagedRow.appendDecimal256(col, value); } - private void appendDecimal64Column(CharSequence name, Decimal64 value, boolean addJournal) { + private void appendDecimal64Column(CharSequence name, Decimal64 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DECIMAL64; - e.column = col; - e.objectValue = value; - } + stagedRow.appendDecimal64(col, value); } - private void appendDesignatedTimestamp(long value, boolean nanos, boolean addJournal) { + private void appendDesignatedTimestamp(long value, boolean nanos) { QwpTableBuffer.ColumnBuffer col; if (nanos) { if (cachedTimestampNanosColumn == null) { @@ -504,111 +482,63 @@ private void appendDesignatedTimestamp(long value, boolean nanos, boolean addJou col = cachedTimestampColumn; } addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS; - e.column = col; - e.longValue = value; - } + stagedRow.appendLong(col, nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS, value); } - private void appendDoubleArrayColumn(CharSequence name, Object value, boolean addJournal) { + private void appendDoubleArrayColumn(CharSequence name, Object value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DOUBLE_ARRAY; - e.column = col; - e.objectValue = value; - } + stagedRow.appendDoubleArray(col, value, estimateDoubleArrayPayload(value)); } - private void appendDoubleColumn(CharSequence name, double value, boolean addJournal) { + private void appendDoubleColumn(CharSequence name, double value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_DOUBLE; - e.column = col; - e.doubleValue = value; - } + stagedRow.appendDouble(col, value); } - private void appendLongArrayColumn(CharSequence name, Object value, boolean addJournal) { + private void appendLongArrayColumn(CharSequence name, Object value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_LONG_ARRAY; - e.column = col; - e.objectValue = value; - } + stagedRow.appendLongArray(col, value, estimateLongArrayPayload(value)); } - private void appendLongColumn(CharSequence name, long value, boolean addJournal) { + private void appendLongColumn(CharSequence name, long value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_LONG; - e.column = col; - e.longValue = value; - } + stagedRow.appendLong(col, ENTRY_LONG, value); } - private void appendStringColumn(CharSequence name, CharSequence value, boolean addJournal) { + private void appendStringColumn(CharSequence name, CharSequence value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_STRING; - e.column = col; - e.stringValue = Chars.toString(value); - } + stagedRow.appendUtf8(col, ENTRY_STRING, value, 0); } - private void appendSymbolColumn(CharSequence name, CharSequence value, boolean addJournal) { + private void appendSymbolColumn(CharSequence name, CharSequence value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = ENTRY_SYMBOL; - e.column = col; - e.stringValue = Chars.toString(value); - } + stagedRow.appendUtf8(col, ENTRY_SYMBOL, value, estimateSymbolPayloadDelta(col, col.getValueCount(), value)); } - private void appendTimestampColumn(CharSequence name, byte type, long value, byte journalKind, boolean addJournal) { + private void appendTimestampColumn(CharSequence name, byte type, long value, byte journalKind) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); currentRowColumnCount++; addTouchedColumn(col); - - if (addJournal) { - ColumnEntry e = nextJournalEntry(); - e.kind = journalKind; - e.column = col; - e.longValue = value; - } + stagedRow.appendLong(col, journalKind, value); } private void atMicros(long timestampMicros) { if (currentRowColumnCount == 0) { throw new LineSenderException("no columns were provided"); } - appendDesignatedTimestamp(timestampMicros, false, true); + appendDesignatedTimestamp(timestampMicros, false); commitCurrentRow(); } @@ -616,7 +546,7 @@ private void atNanos(long timestampNanos) { if (currentRowColumnCount == 0) { throw new LineSenderException("no columns were provided"); } - appendDesignatedTimestamp(timestampNanos, true, true); + appendDesignatedTimestamp(timestampNanos, true); commitCurrentRow(); } @@ -633,13 +563,7 @@ private void checkTableSelected() { } private void clearStagedRow() { - for (int i = 0; i < rowJournalSize; i++) { - ColumnEntry entry = rowJournal.getQuick(i); - entry.column = null; - entry.objectValue = null; - entry.stringValue = null; - } - rowJournalSize = 0; + stagedRow.clear(); currentRowColumnCount = 0; touchedColumnCount = 0; missingColumnCount = 0; @@ -753,10 +677,9 @@ private long estimateBaseForCurrentSchema() { private long estimateCurrentDatagramSizeWithStagedRow(int targetRows) { long estimate = currentTableBuffer.getRowCount() > 0 ? committedEstimate : estimateBaseForCurrentSchema(); - for (int i = 0; i < rowJournalSize; i++) { - ColumnEntry entry = rowJournal.getQuick(i); - QwpTableBuffer.ColumnBuffer col = entry.column; - estimate += estimateEntryPayload(entry, col); + for (int i = 0, n = stagedRow.size(); i < n; i++) { + QwpTableBuffer.ColumnBuffer col = stagedRow.getColumn(i); + estimate += estimateEntryPayload(i, col); if (col.isNullable()) { estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); } @@ -773,20 +696,19 @@ private long estimateCurrentDatagramSizeWithStagedRow(int targetRows) { return estimate; } - private long estimateEntryPayload(ColumnEntry entry, QwpTableBuffer.ColumnBuffer col) { + private long estimateEntryPayload(int entryIndex, QwpTableBuffer.ColumnBuffer col) { int valueCountBefore = col.getValueCount(); - return switch (entry.kind) { + return switch (stagedRow.getKind(entryIndex)) { case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_DOUBLE, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> 8; case ENTRY_BOOL -> packedBytes(valueCountBefore + 1) - packedBytes(valueCountBefore); case ENTRY_DECIMAL64 -> 8; case ENTRY_DECIMAL128 -> 16; case ENTRY_DECIMAL256 -> 32; - case ENTRY_DOUBLE_ARRAY -> estimateDoubleArrayPayload(entry.objectValue); - case ENTRY_LONG_ARRAY -> estimateLongArrayPayload(entry.objectValue); - case ENTRY_STRING -> entry.stringValue == null ? 0 : 4L + utf8Length(entry.stringValue); - case ENTRY_SYMBOL -> estimateSymbolPayloadDelta(col, valueCountBefore, entry.stringValue); - default -> throw new LineSenderException("unknown staged row entry type: " + entry.kind); + case ENTRY_DOUBLE_ARRAY, ENTRY_LONG_ARRAY -> stagedRow.getLong0(entryIndex); + case ENTRY_STRING -> stagedRow.getAuxInt(entryIndex) < 0 ? 0 : 4L + stagedRow.getAuxInt(entryIndex); + case ENTRY_SYMBOL -> stagedRow.getLong1(entryIndex); + default -> throw new LineSenderException("unknown staged row entry type: " + stagedRow.getKind(entryIndex)); }; } @@ -929,7 +851,7 @@ private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { } private boolean hasInProgressRow() { - return rowJournalSize > 0; + return stagedRow.size() > 0; } private boolean isTouchedColumn(QwpTableBuffer.ColumnBuffer col) { @@ -950,28 +872,10 @@ private void addTouchedColumn(QwpTableBuffer.ColumnBuffer col) { } private void materializeCurrentRow() { - for (int i = 0; i < rowJournalSize; i++) { - ColumnEntry entry = rowJournal.getQuick(i); - QwpTableBuffer.ColumnBuffer col = entry.column; - switch (entry.kind) { - case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> - col.addLong(entry.longValue); - case ENTRY_BOOL -> col.addBoolean(entry.boolValue); - case ENTRY_DECIMAL64 -> col.addDecimal64((Decimal64) entry.objectValue); - case ENTRY_DECIMAL128 -> col.addDecimal128((Decimal128) entry.objectValue); - case ENTRY_DECIMAL256 -> col.addDecimal256((Decimal256) entry.objectValue); - case ENTRY_DOUBLE -> col.addDouble(entry.doubleValue); - case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayValue(col, entry.objectValue); - case ENTRY_LONG_ARRAY -> appendLongArrayValue(col, entry.objectValue); - case ENTRY_STRING -> col.addString(entry.stringValue); - case ENTRY_SYMBOL -> col.addSymbol(entry.stringValue); - default -> throw new LineSenderException("unknown staged row entry type: " + entry.kind); - } - } - currentTableBuffer.nextRow(missingColumns, missingColumnCount); + stagedRow.materializeInto(currentTableBuffer, missingColumns, missingColumnCount); } - private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { + private static void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { if (value instanceof double[] values) { col.addDoubleArray(values); } else if (value instanceof double[][] values) { @@ -985,7 +889,7 @@ private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Object valu } } - private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { + private static void appendLongArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { if (value instanceof long[] values) { col.addLongArray(values); } else if (value instanceof long[][] values) { @@ -1022,21 +926,6 @@ private static int packedBytes(int valueCount) { return (valueCount + 7) / 8; } - private ColumnEntry nextJournalEntry() { - if (rowJournalSize < rowJournal.size()) { - ColumnEntry entry = rowJournal.getQuick(rowJournalSize); - entry.column = null; - entry.objectValue = null; - entry.stringValue = null; - rowJournalSize++; - return entry; - } - ColumnEntry entry = new ColumnEntry(); - rowJournal.add(entry); - rowJournalSize++; - return entry; - } - private void resetCommittedEstimate() { committedEstimate = 0; } @@ -1149,13 +1038,206 @@ private void reset() { } } - private static class ColumnEntry { - boolean boolValue; - QwpTableBuffer.ColumnBuffer column; - double doubleValue; - byte kind; - long longValue; - Object objectValue; - String stringValue; + private static final class NativeRowStaging { + private static final int AUX_INT_OFFSET = 4; + private static final int ENTRY_SIZE = 40; + private static final int LONG0_OFFSET = 8; + private static final int LONG1_OFFSET = 16; + private static final int LONG2_OFFSET = 24; + private static final int LONG3_OFFSET = 32; + + private final Decimal128 decimal128Sink = new Decimal128(); + private final Decimal256 decimal256Sink = new Decimal256(); + private final Decimal64 decimal64Sink = new Decimal64(); + private final OffHeapAppendMemory entries = new OffHeapAppendMemory(ENTRY_SIZE * 8L); + private final OffHeapAppendMemory varData = new OffHeapAppendMemory(128); + private QwpTableBuffer.ColumnBuffer[] columns = new QwpTableBuffer.ColumnBuffer[8]; + private int size; + private Object[] sidecarObjects = new Object[8]; + + void appendBoolean(QwpTableBuffer.ColumnBuffer column, boolean value) { + appendEntry(column, null, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0); + } + + void appendDecimal128(QwpTableBuffer.ColumnBuffer column, Decimal128 value) { + appendEntry(column, null, ENTRY_DECIMAL128, value.getScale(), value.getHigh(), value.getLow(), 0, 0); + } + + void appendDecimal256(QwpTableBuffer.ColumnBuffer column, Decimal256 value) { + appendEntry( + column, + null, + ENTRY_DECIMAL256, + value.getScale(), + value.getHh(), + value.getHl(), + value.getLh(), + value.getLl() + ); + } + + void appendDecimal64(QwpTableBuffer.ColumnBuffer column, Decimal64 value) { + appendEntry(column, null, ENTRY_DECIMAL64, value.getScale(), value.getValue(), 0, 0, 0); + } + + void appendDouble(QwpTableBuffer.ColumnBuffer column, double value) { + appendEntry(column, null, ENTRY_DOUBLE, 0, Double.doubleToRawLongBits(value), 0, 0, 0); + } + + void appendDoubleArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) { + appendEntry(column, value, ENTRY_DOUBLE_ARRAY, 0, estimatePayload, 0, 0, 0); + } + + void appendLong(QwpTableBuffer.ColumnBuffer column, int kind, long value) { + appendEntry(column, null, kind, 0, value, 0, 0, 0); + } + + void appendLongArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) { + appendEntry(column, value, ENTRY_LONG_ARRAY, 0, estimatePayload, 0, 0, 0); + } + + void appendUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value, long long1) { + int len = -1; + long offset = 0; + if (value != null) { + offset = varData.getAppendOffset(); + varData.putUtf8(value); + len = (int) (varData.getAppendOffset() - offset); + } + appendEntry(column, null, kind, len, offset, long1, 0, 0); + } + + void clear() { + for (int i = 0; i < size; i++) { + columns[i] = null; + sidecarObjects[i] = null; + } + size = 0; + entries.truncate(); + varData.truncate(); + } + + void close() { + entries.close(); + varData.close(); + } + + int getAuxInt(int index) { + return Unsafe.getUnsafe().getInt(entryAddress(index) + AUX_INT_OFFSET); + } + + QwpTableBuffer.ColumnBuffer getColumn(int index) { + return columns[index]; + } + + int getKind(int index) { + return Unsafe.getUnsafe().getInt(entryAddress(index)); + } + + long getLong0(int index) { + return Unsafe.getUnsafe().getLong(entryAddress(index) + LONG0_OFFSET); + } + + long getLong1(int index) { + return Unsafe.getUnsafe().getLong(entryAddress(index) + LONG1_OFFSET); + } + + void materializeInto(QwpTableBuffer tableBuffer, QwpTableBuffer.ColumnBuffer[] missingColumns, int missingColumnCount) { + for (int i = 0; i < size; i++) { + QwpTableBuffer.ColumnBuffer column = columns[i]; + long entryAddress = entryAddress(i); + int kind = Unsafe.getUnsafe().getInt(entryAddress); + int auxInt = Unsafe.getUnsafe().getInt(entryAddress + AUX_INT_OFFSET); + long long0 = Unsafe.getUnsafe().getLong(entryAddress + LONG0_OFFSET); + long long1 = Unsafe.getUnsafe().getLong(entryAddress + LONG1_OFFSET); + switch (kind) { + case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> + column.addLong(long0); + case ENTRY_BOOL -> column.addBoolean(long0 != 0); + case ENTRY_DECIMAL64 -> { + decimal64Sink.ofRaw(long0); + decimal64Sink.setScale(auxInt); + column.addDecimal64(decimal64Sink); + } + case ENTRY_DECIMAL128 -> { + decimal128Sink.ofRaw(long0, long1); + decimal128Sink.setScale(auxInt); + column.addDecimal128(decimal128Sink); + } + case ENTRY_DECIMAL256 -> { + decimal256Sink.ofRaw( + long0, + long1, + Unsafe.getUnsafe().getLong(entryAddress + LONG2_OFFSET), + Unsafe.getUnsafe().getLong(entryAddress + LONG3_OFFSET) + ); + decimal256Sink.setScale(auxInt); + column.addDecimal256(decimal256Sink); + } + case ENTRY_DOUBLE -> column.addDouble(Double.longBitsToDouble(long0)); + case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayValue(column, sidecarObjects[i]); + case ENTRY_LONG_ARRAY -> appendLongArrayValue(column, sidecarObjects[i]); + case ENTRY_STRING -> column.addStringUtf8(varData.addressOf(long0), auxInt); + case ENTRY_SYMBOL -> { + if (auxInt < 0) { + column.addSymbol(null); + } else { + column.addSymbolUtf8(varData.addressOf(long0), auxInt); + } + } + default -> throw new LineSenderException("unknown staged row entry type: " + kind); + } + } + tableBuffer.nextRow(missingColumns, missingColumnCount); + } + + int size() { + return size; + } + + private void appendEntry( + QwpTableBuffer.ColumnBuffer column, + Object sidecarObject, + int kind, + int auxInt, + long long0, + long long1, + long long2, + long long3 + ) { + ensureCapacity(size + 1); + columns[size] = column; + sidecarObjects[size] = sidecarObject; + entries.putInt(kind); + entries.putInt(auxInt); + entries.putLong(long0); + entries.putLong(long1); + entries.putLong(long2); + entries.putLong(long3); + size++; + } + + private long entryAddress(int index) { + return entries.addressOf((long) index * ENTRY_SIZE); + } + + private void ensureCapacity(int required) { + if (required <= columns.length) { + return; + } + + int newCapacity = columns.length; + while (newCapacity < required) { + newCapacity *= 2; + } + + QwpTableBuffer.ColumnBuffer[] newColumns = new QwpTableBuffer.ColumnBuffer[newCapacity]; + System.arraycopy(columns, 0, newColumns, 0, size); + columns = newColumns; + + Object[] newSidecarObjects = new Object[newCapacity]; + System.arraycopy(sidecarObjects, 0, newSidecarObjects, 0, size); + sidecarObjects = newSidecarObjects; + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index 4d418f4..6d92e57 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -27,6 +27,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; /** * Lightweight append-only off-heap buffer for columnar data storage. @@ -104,6 +105,15 @@ public void putByte(byte value) { appendAddress++; } + public void putBlockOfBytes(long from, long len) { + if (len <= 0) { + return; + } + ensureCapacity(len); + Vect.memcpy(appendAddress, from, len); + appendAddress += len; + } + public void putDouble(double value) { ensureCapacity(8); Unsafe.getUnsafe().putDouble(appendAddress, value); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 2f72c14..9bddd9b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -40,6 +40,7 @@ import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; +import io.questdb.client.std.str.Utf8s; import java.util.Arrays; @@ -971,6 +972,21 @@ public void addString(CharSequence value) { size++; } + public void addStringUtf8(long ptr, int len) { + if (len < 0 && nullable) { + ensureNullCapacity(size + 1); + markNull(size); + } else { + ensureNullBitmapForNonNull(); + if (len > 0) { + stringData.putBlockOfBytes(ptr, len); + } + stringOffsets.putInt((int) stringData.getAppendOffset()); + valueCount++; + } + size++; + } + public void addSymbol(CharSequence value) { if (value == null) { addNull(); @@ -988,6 +1004,14 @@ public void addSymbol(CharSequence value) { size++; } + public void addSymbolUtf8(long ptr, int len) { + if (len < 0) { + addNull(); + return; + } + addSymbol(Utf8s.stringFromUtf8Bytes(ptr, ptr + len)); + } + public void addSymbolWithGlobalId(String value, int globalId) { if (value == null) { addNull(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index faefe4d..f2bf425 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -676,6 +676,57 @@ public void testCancelRowAfterMidRowSchemaChangeDoesNotLeakSchema() throws Excep }); } + @Test + public void testUtf8StringAndSymbolStagingSupportsCancelAndPacketSizing() throws Exception { + assertMemoryLeak(() -> { + String msg1 = "Gruesse 東京"; + String msg2 = "Privet 👋 kosme"; + List rows = Arrays.asList( + row("utf8", sender -> sender.table("utf8") + .longColumn("x", 1) + .symbol("sym", "東京") + .stringColumn("msg", msg1) + .atNow(), + "x", 1L, + "sym", "東京", + "msg", msg1), + row("utf8", sender -> sender.table("utf8") + .longColumn("x", 2) + .symbol("sym", "Αθηνα") + .stringColumn("msg", msg2) + .atNow(), + "x", 2L, + "sym", "Αθηνα", + "msg", msg2) + ); + int maxDatagramSize = fullPacketSize(rows) - 1; + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("utf8") + .longColumn("x", 0) + .symbol("sym", "キャンセル") + .stringColumn("msg", "should not ship 👎"); + sender.cancelRow(); + + sender.longColumn("x", 1) + .symbol("sym", "東京") + .stringColumn("msg", msg1) + .atNow(); + sender.longColumn("x", 2) + .symbol("sym", "Αθηνα") + .stringColumn("msg", msg2) + .atNow(); + sender.flush(); + } + + RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount); + Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(nf.packets)); + }); + } + @Test public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() throws Exception { assertMemoryLeak(() -> { From 5fd10f6635dbc1e946141eb359adec7a5a41a6c2 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 6 Mar 2026 15:05:49 +0100 Subject: [PATCH 164/230] cleanup --- .../cutlass/qwp/client/QwpUdpSender.java | 322 +++++++++--------- 1 file changed, 161 insertions(+), 161 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index ffc6cf3..56fa21c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -87,23 +87,23 @@ public class QwpUdpSender implements Sender { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private final NativeBufferWriter headerBuffer = new NativeBufferWriter(); private final int maxDatagramSize; - private final SegmentedNativeBufferWriter payloadBuffer = new SegmentedNativeBufferWriter(); + private final SegmentedNativeBufferWriter payloadWriter = new SegmentedNativeBufferWriter(); private final NativeRowStaging stagedRow = new NativeRowStaging(); private final boolean trackDatagramEstimate; - private final NativeSegmentList sendSegments = new NativeSegmentList(); + private final NativeSegmentList datagramSegments = new NativeSegmentList(); private final CharSequenceObjHashMap tableBuffers; - private QwpTableBuffer.ColumnBuffer[] touchedColumns = new QwpTableBuffer.ColumnBuffer[8]; + private QwpTableBuffer.ColumnBuffer[] stagedColumns = new QwpTableBuffer.ColumnBuffer[8]; private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; private boolean closed; - private long committedEstimate; - private int currentRowColumnCount; + private long committedDatagramEstimate; + private int stagedRowValueCount; private QwpTableBuffer currentTableBuffer; private String currentTableName; - private QwpTableBuffer.ColumnBuffer[] missingColumns = new QwpTableBuffer.ColumnBuffer[8]; - private int missingColumnCount; - private int touchedColumnCount; + private QwpTableBuffer.ColumnBuffer[] rowFillColumns = new QwpTableBuffer.ColumnBuffer[8]; + private int rowFillColumnCount; + private int stagedColumnCount; public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { this(nf, interfaceIPv4, sendToAddress, port, ttl, 0); @@ -147,7 +147,7 @@ public void atNow() { public Sender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - appendBooleanColumn(columnName, value); + stageBooleanColumnValue(columnName, value); return this; } @@ -196,8 +196,8 @@ public void close() { } tableBuffers.clear(); channel.close(); - payloadBuffer.close(); - sendSegments.close(); + payloadWriter.close(); + datagramSegments.close(); headerBuffer.close(); stagedRow.close(); } @@ -210,7 +210,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { } checkNotClosed(); checkTableSelected(); - appendDecimal64Column(name, value); + stageDecimal64ColumnValue(name, value); return this; } @@ -221,7 +221,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { } checkNotClosed(); checkTableSelected(); - appendDecimal128Column(name, value); + stageDecimal128ColumnValue(name, value); return this; } @@ -232,7 +232,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { } checkNotClosed(); checkTableSelected(); - appendDecimal256Column(name, value); + stageDecimal256ColumnValue(name, value); return this; } @@ -243,7 +243,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, values); + stageDoubleArrayColumnValue(name, values); return this; } @@ -254,7 +254,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, values); + stageDoubleArrayColumnValue(name, values); return this; } @@ -265,7 +265,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, values); + stageDoubleArrayColumnValue(name, values); return this; } @@ -276,7 +276,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { } checkNotClosed(); checkTableSelected(); - appendDoubleArrayColumn(name, array); + stageDoubleArrayColumnValue(name, array); return this; } @@ -284,7 +284,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public Sender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - appendDoubleColumn(columnName, value); + stageDoubleColumnValue(columnName, value); return this; } @@ -302,7 +302,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, values); + stageLongArrayColumnValue(name, values); return this; } @@ -313,7 +313,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, values); + stageLongArrayColumnValue(name, values); return this; } @@ -324,7 +324,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, values); + stageLongArrayColumnValue(name, values); return this; } @@ -335,7 +335,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { } checkNotClosed(); checkTableSelected(); - appendLongArrayColumn(name, array); + stageLongArrayColumnValue(name, array); return this; } @@ -343,7 +343,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public Sender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - appendLongColumn(columnName, value); + stageLongColumnValue(columnName, value); return this; } @@ -363,14 +363,14 @@ public void reset() { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; clearStagedRow(); - resetCommittedEstimate(); + resetCommittedDatagramEstimate(); } @Override public Sender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - appendStringColumn(columnName, value); + stageStringColumnValue(columnName, value); return this; } @@ -378,7 +378,7 @@ public Sender stringColumn(CharSequence columnName, CharSequence value) { public Sender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - appendSymbolColumn(columnName, value); + stageSymbolColumnValue(columnName, value); return this; } @@ -395,7 +395,7 @@ public Sender table(CharSequence tableName) { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; clearStagedRow(); - resetCommittedEstimate(); + resetCommittedDatagramEstimate(); currentTableName = tableName.toString(); currentTableBuffer = tableBuffers.get(currentTableName); @@ -411,10 +411,10 @@ public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit un checkNotClosed(); checkTableSelected(); if (unit == ChronoUnit.NANOS) { - appendTimestampColumn(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS); + stageTimestampColumnValue(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS); } else { long micros = toMicros(value, unit); - appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS); + stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS); } return this; } @@ -424,7 +424,7 @@ public Sender timestampColumn(CharSequence columnName, Instant value) { checkNotClosed(); checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - appendTimestampColumn(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS); + stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS); return this; } @@ -440,35 +440,35 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, return col; } - private void appendBooleanColumn(CharSequence name, boolean value) { + private void stageBooleanColumnValue(CharSequence name, boolean value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendBoolean(col, value); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageBoolean(col, value); } - private void appendDecimal128Column(CharSequence name, Decimal128 value) { + private void stageDecimal128ColumnValue(CharSequence name, Decimal128 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendDecimal128(col, value); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageDecimal128(col, value); } - private void appendDecimal256Column(CharSequence name, Decimal256 value) { + private void stageDecimal256ColumnValue(CharSequence name, Decimal256 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendDecimal256(col, value); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageDecimal256(col, value); } - private void appendDecimal64Column(CharSequence name, Decimal64 value) { + private void stageDecimal64ColumnValue(CharSequence name, Decimal64 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendDecimal64(col, value); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageDecimal64(col, value); } - private void appendDesignatedTimestamp(long value, boolean nanos) { + private void stageDesignatedTimestampValue(long value, boolean nanos) { QwpTableBuffer.ColumnBuffer col; if (nanos) { if (cachedTimestampNanosColumn == null) { @@ -481,72 +481,72 @@ private void appendDesignatedTimestamp(long value, boolean nanos) { } col = cachedTimestampColumn; } - addTouchedColumn(col); - stagedRow.appendLong(col, nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS, value); + addStagedColumn(col); + stagedRow.stageLong(col, nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS, value); } - private void appendDoubleArrayColumn(CharSequence name, Object value) { + private void stageDoubleArrayColumnValue(CharSequence name, Object value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendDoubleArray(col, value, estimateDoubleArrayPayload(value)); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageDoubleArray(col, value, estimateDoubleArrayPayload(value)); } - private void appendDoubleColumn(CharSequence name, double value) { + private void stageDoubleColumnValue(CharSequence name, double value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendDouble(col, value); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageDouble(col, value); } - private void appendLongArrayColumn(CharSequence name, Object value) { + private void stageLongArrayColumnValue(CharSequence name, Object value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendLongArray(col, value, estimateLongArrayPayload(value)); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageLongArray(col, value, estimateLongArrayPayload(value)); } - private void appendLongColumn(CharSequence name, long value) { + private void stageLongColumnValue(CharSequence name, long value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendLong(col, ENTRY_LONG, value); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageLong(col, ENTRY_LONG, value); } - private void appendStringColumn(CharSequence name, CharSequence value) { + private void stageStringColumnValue(CharSequence name, CharSequence value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendUtf8(col, ENTRY_STRING, value, 0); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageUtf8(col, ENTRY_STRING, value, 0); } - private void appendSymbolColumn(CharSequence name, CharSequence value) { + private void stageSymbolColumnValue(CharSequence name, CharSequence value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendUtf8(col, ENTRY_SYMBOL, value, estimateSymbolPayloadDelta(col, col.getValueCount(), value)); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageUtf8(col, ENTRY_SYMBOL, value, estimateSymbolPayloadDelta(col, col.getValueCount(), value)); } - private void appendTimestampColumn(CharSequence name, byte type, long value, byte journalKind) { + private void stageTimestampColumnValue(CharSequence name, byte type, long value, byte entryKind) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); - currentRowColumnCount++; - addTouchedColumn(col); - stagedRow.appendLong(col, journalKind, value); + stagedRowValueCount++; + addStagedColumn(col); + stagedRow.stageLong(col, entryKind, value); } private void atMicros(long timestampMicros) { - if (currentRowColumnCount == 0) { + if (stagedRowValueCount == 0) { throw new LineSenderException("no columns were provided"); } - appendDesignatedTimestamp(timestampMicros, false); + stageDesignatedTimestampValue(timestampMicros, false); commitCurrentRow(); } private void atNanos(long timestampNanos) { - if (currentRowColumnCount == 0) { + if (stagedRowValueCount == 0) { throw new LineSenderException("no columns were provided"); } - appendDesignatedTimestamp(timestampNanos, true); + stageDesignatedTimestampValue(timestampNanos, true); commitCurrentRow(); } @@ -564,34 +564,34 @@ private void checkTableSelected() { private void clearStagedRow() { stagedRow.clear(); - currentRowColumnCount = 0; - touchedColumnCount = 0; - missingColumnCount = 0; + stagedRowValueCount = 0; + stagedColumnCount = 0; + rowFillColumnCount = 0; } - private void collectMissingColumns(int targetRows) { - missingColumnCount = 0; + private void collectRowFillColumns(int targetRows) { + rowFillColumnCount = 0; for (int i = 0, n = currentTableBuffer.getColumnCount(); i < n; i++) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); - if (isTouchedColumn(col)) { + if (isStagedColumn(col)) { continue; } if (col.getSize() >= targetRows) { continue; } - ensureMissingColumnCapacity(missingColumnCount + 1); - missingColumns[missingColumnCount++] = col; + ensureRowFillColumnCapacity(rowFillColumnCount + 1); + rowFillColumns[rowFillColumnCount++] = col; } } private void commitCurrentRow() { - if (currentRowColumnCount == 0) { + if (stagedRowValueCount == 0) { throw new LineSenderException("no columns were provided"); } long estimate = 0; int targetRows = currentTableBuffer.getRowCount() + 1; - collectMissingColumns(targetRows); + collectRowFillColumns(targetRows); if (trackDatagramEstimate) { estimate = estimateCurrentDatagramSizeWithStagedRow(targetRows); if (estimate > maxDatagramSize) { @@ -600,7 +600,7 @@ private void commitCurrentRow() { } flushCommittedRowsOfCurrentTable(); targetRows = currentTableBuffer.getRowCount() + 1; - collectMissingColumns(targetRows); + collectRowFillColumns(targetRows); estimate = estimateCurrentDatagramSizeWithStagedRow(targetRows); if (estimate > maxDatagramSize) { throw singleRowTooLarge(estimate); @@ -608,9 +608,9 @@ private void commitCurrentRow() { } } - materializeCurrentRow(); + commitStagedRow(); if (trackDatagramEstimate) { - committedEstimate = estimate; + committedDatagramEstimate = estimate; } clearStagedRow(); } @@ -624,12 +624,12 @@ private void ensureNoInProgressRow(String operation) { } } - private int encodePayloadForUdp(QwpTableBuffer tableBuffer) { - payloadBuffer.reset(); - columnWriter.setBuffer(payloadBuffer); + private int encodeTablePayloadForUdp(QwpTableBuffer tableBuffer) { + payloadWriter.reset(); + columnWriter.setBuffer(payloadWriter); columnWriter.encodeTable(tableBuffer, false, false, false); - payloadBuffer.finish(); - return payloadBuffer.getPosition(); + payloadWriter.finish(); + return payloadWriter.getPosition(); } private long estimateArrayValueSize(int nDims, long elementCount) { @@ -676,16 +676,16 @@ private long estimateBaseForCurrentSchema() { } private long estimateCurrentDatagramSizeWithStagedRow(int targetRows) { - long estimate = currentTableBuffer.getRowCount() > 0 ? committedEstimate : estimateBaseForCurrentSchema(); + long estimate = currentTableBuffer.getRowCount() > 0 ? committedDatagramEstimate : estimateBaseForCurrentSchema(); for (int i = 0, n = stagedRow.size(); i < n; i++) { QwpTableBuffer.ColumnBuffer col = stagedRow.getColumn(i); - estimate += estimateEntryPayload(i, col); + estimate += estimateStagedEntryPayload(i, col); if (col.isNullable()) { estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); } } - for (int i = 0; i < missingColumnCount; i++) { - QwpTableBuffer.ColumnBuffer col = missingColumns[i]; + for (int i = 0; i < rowFillColumnCount; i++) { + QwpTableBuffer.ColumnBuffer col = rowFillColumns[i]; int missing = targetRows - col.getSize(); if (col.isNullable()) { estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); @@ -696,7 +696,7 @@ private long estimateCurrentDatagramSizeWithStagedRow(int targetRows) { return estimate; } - private long estimateEntryPayload(int entryIndex, QwpTableBuffer.ColumnBuffer col) { + private long estimateStagedEntryPayload(int entryIndex, QwpTableBuffer.ColumnBuffer col) { int valueCountBefore = col.getValueCount(); return switch (stagedRow.getKind(entryIndex)) { case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_DOUBLE, ENTRY_LONG, @@ -754,34 +754,34 @@ private long estimateLongArrayPayload(Object value) { throw new LineSenderException("unsupported long array type"); } - private void ensureMissingColumnCapacity(int required) { - if (required <= missingColumns.length) { + private void ensureRowFillColumnCapacity(int required) { + if (required <= rowFillColumns.length) { return; } - int newCapacity = missingColumns.length; + int newCapacity = rowFillColumns.length; while (newCapacity < required) { newCapacity *= 2; } QwpTableBuffer.ColumnBuffer[] newArr = new QwpTableBuffer.ColumnBuffer[newCapacity]; - System.arraycopy(missingColumns, 0, newArr, 0, missingColumnCount); - missingColumns = newArr; + System.arraycopy(rowFillColumns, 0, newArr, 0, rowFillColumnCount); + rowFillColumns = newArr; } - private void ensureTouchedColumnCapacity(int required) { - if (required <= touchedColumns.length) { + private void ensureStagedColumnCapacity(int required) { + if (required <= stagedColumns.length) { return; } - int newCapacity = touchedColumns.length; + int newCapacity = stagedColumns.length; while (newCapacity < required) { newCapacity *= 2; } QwpTableBuffer.ColumnBuffer[] newArr = new QwpTableBuffer.ColumnBuffer[newCapacity]; - System.arraycopy(touchedColumns, 0, newArr, 0, touchedColumnCount); - touchedColumns = newArr; + System.arraycopy(stagedColumns, 0, newArr, 0, stagedColumnCount); + stagedColumns = newArr; } private long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, int valueCountBefore, CharSequence value) { @@ -820,7 +820,7 @@ private void flushCommittedRowsOfCurrentTable() { sendTableBuffer(currentTableName, currentTableBuffer); cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - resetCommittedEstimate(); + resetCommittedDatagramEstimate(); } private void flushInternal() { @@ -839,7 +839,7 @@ private void flushInternal() { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; clearStagedRow(); - resetCommittedEstimate(); + resetCommittedDatagramEstimate(); } private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { @@ -847,35 +847,35 @@ private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; clearStagedRow(); - resetCommittedEstimate(); + resetCommittedDatagramEstimate(); } private boolean hasInProgressRow() { return stagedRow.size() > 0; } - private boolean isTouchedColumn(QwpTableBuffer.ColumnBuffer col) { - for (int i = 0; i < touchedColumnCount; i++) { - if (touchedColumns[i] == col) { + private boolean isStagedColumn(QwpTableBuffer.ColumnBuffer col) { + for (int i = 0; i < stagedColumnCount; i++) { + if (stagedColumns[i] == col) { return true; } } return false; } - private void addTouchedColumn(QwpTableBuffer.ColumnBuffer col) { - if (isTouchedColumn(col)) { + private void addStagedColumn(QwpTableBuffer.ColumnBuffer col) { + if (isStagedColumn(col)) { return; } - ensureTouchedColumnCapacity(touchedColumnCount + 1); - touchedColumns[touchedColumnCount++] = col; + ensureStagedColumnCapacity(stagedColumnCount + 1); + stagedColumns[stagedColumnCount++] = col; } - private void materializeCurrentRow() { - stagedRow.materializeInto(currentTableBuffer, missingColumns, missingColumnCount); + private void commitStagedRow() { + stagedRow.commitIntoTableBuffer(currentTableBuffer, rowFillColumns, rowFillColumnCount); } - private static void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { + private static void commitDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { if (value instanceof double[] values) { col.addDoubleArray(values); } else if (value instanceof double[][] values) { @@ -889,7 +889,7 @@ private static void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Obje } } - private static void appendLongArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { + private static void commitLongArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) { if (value instanceof long[] values) { col.addLongArray(values); } else if (value instanceof long[][] values) { @@ -926,12 +926,12 @@ private static int packedBytes(int valueCount) { return (valueCount + 7) / 8; } - private void resetCommittedEstimate() { - committedEstimate = 0; + private void resetCommittedDatagramEstimate() { + committedDatagramEstimate = 0; } private void sendTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer) { - int payloadLength = encodePayloadForUdp(tableBuffer); + int payloadLength = encodeTablePayloadForUdp(tableBuffer); headerBuffer.reset(); headerBuffer.putByte((byte) 'Q'); headerBuffer.putByte((byte) 'W'); @@ -942,15 +942,15 @@ private void sendTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer) headerBuffer.putShort((short) 1); headerBuffer.putInt(payloadLength); - sendSegments.reset(); - sendSegments.add(headerBuffer.getBufferPtr(), headerBuffer.getPosition()); - sendSegments.appendFrom(payloadBuffer.getSegments()); + datagramSegments.reset(); + datagramSegments.add(headerBuffer.getBufferPtr(), headerBuffer.getPosition()); + datagramSegments.appendFrom(payloadWriter.getSegments()); try { channel.sendSegments( - sendSegments.getAddress(), - sendSegments.getSegmentCount(), - (int) sendSegments.getTotalLength() + datagramSegments.getAddress(), + datagramSegments.getSegmentCount(), + (int) datagramSegments.getTotalLength() ); } catch (LineSenderException e) { LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e)); @@ -1055,16 +1055,16 @@ private static final class NativeRowStaging { private int size; private Object[] sidecarObjects = new Object[8]; - void appendBoolean(QwpTableBuffer.ColumnBuffer column, boolean value) { - appendEntry(column, null, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0); + void stageBoolean(QwpTableBuffer.ColumnBuffer column, boolean value) { + stageEntry(column, null, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0); } - void appendDecimal128(QwpTableBuffer.ColumnBuffer column, Decimal128 value) { - appendEntry(column, null, ENTRY_DECIMAL128, value.getScale(), value.getHigh(), value.getLow(), 0, 0); + void stageDecimal128(QwpTableBuffer.ColumnBuffer column, Decimal128 value) { + stageEntry(column, null, ENTRY_DECIMAL128, value.getScale(), value.getHigh(), value.getLow(), 0, 0); } - void appendDecimal256(QwpTableBuffer.ColumnBuffer column, Decimal256 value) { - appendEntry( + void stageDecimal256(QwpTableBuffer.ColumnBuffer column, Decimal256 value) { + stageEntry( column, null, ENTRY_DECIMAL256, @@ -1076,27 +1076,27 @@ void appendDecimal256(QwpTableBuffer.ColumnBuffer column, Decimal256 value) { ); } - void appendDecimal64(QwpTableBuffer.ColumnBuffer column, Decimal64 value) { - appendEntry(column, null, ENTRY_DECIMAL64, value.getScale(), value.getValue(), 0, 0, 0); + void stageDecimal64(QwpTableBuffer.ColumnBuffer column, Decimal64 value) { + stageEntry(column, null, ENTRY_DECIMAL64, value.getScale(), value.getValue(), 0, 0, 0); } - void appendDouble(QwpTableBuffer.ColumnBuffer column, double value) { - appendEntry(column, null, ENTRY_DOUBLE, 0, Double.doubleToRawLongBits(value), 0, 0, 0); + void stageDouble(QwpTableBuffer.ColumnBuffer column, double value) { + stageEntry(column, null, ENTRY_DOUBLE, 0, Double.doubleToRawLongBits(value), 0, 0, 0); } - void appendDoubleArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) { - appendEntry(column, value, ENTRY_DOUBLE_ARRAY, 0, estimatePayload, 0, 0, 0); + void stageDoubleArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) { + stageEntry(column, value, ENTRY_DOUBLE_ARRAY, 0, estimatePayload, 0, 0, 0); } - void appendLong(QwpTableBuffer.ColumnBuffer column, int kind, long value) { - appendEntry(column, null, kind, 0, value, 0, 0, 0); + void stageLong(QwpTableBuffer.ColumnBuffer column, int kind, long value) { + stageEntry(column, null, kind, 0, value, 0, 0, 0); } - void appendLongArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) { - appendEntry(column, value, ENTRY_LONG_ARRAY, 0, estimatePayload, 0, 0, 0); + void stageLongArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) { + stageEntry(column, value, ENTRY_LONG_ARRAY, 0, estimatePayload, 0, 0, 0); } - void appendUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value, long long1) { + void stageUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value, long long1) { int len = -1; long offset = 0; if (value != null) { @@ -1104,7 +1104,7 @@ void appendUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value varData.putUtf8(value); len = (int) (varData.getAppendOffset() - offset); } - appendEntry(column, null, kind, len, offset, long1, 0, 0); + stageEntry(column, null, kind, len, offset, long1, 0, 0); } void clear() { @@ -1142,7 +1142,7 @@ long getLong1(int index) { return Unsafe.getUnsafe().getLong(entryAddress(index) + LONG1_OFFSET); } - void materializeInto(QwpTableBuffer tableBuffer, QwpTableBuffer.ColumnBuffer[] missingColumns, int missingColumnCount) { + void commitIntoTableBuffer(QwpTableBuffer tableBuffer, QwpTableBuffer.ColumnBuffer[] rowFillColumns, int rowFillColumnCount) { for (int i = 0; i < size; i++) { QwpTableBuffer.ColumnBuffer column = columns[i]; long entryAddress = entryAddress(i); @@ -1175,8 +1175,8 @@ void materializeInto(QwpTableBuffer tableBuffer, QwpTableBuffer.ColumnBuffer[] m column.addDecimal256(decimal256Sink); } case ENTRY_DOUBLE -> column.addDouble(Double.longBitsToDouble(long0)); - case ENTRY_DOUBLE_ARRAY -> appendDoubleArrayValue(column, sidecarObjects[i]); - case ENTRY_LONG_ARRAY -> appendLongArrayValue(column, sidecarObjects[i]); + case ENTRY_DOUBLE_ARRAY -> commitDoubleArrayValue(column, sidecarObjects[i]); + case ENTRY_LONG_ARRAY -> commitLongArrayValue(column, sidecarObjects[i]); case ENTRY_STRING -> column.addStringUtf8(varData.addressOf(long0), auxInt); case ENTRY_SYMBOL -> { if (auxInt < 0) { @@ -1188,14 +1188,14 @@ void materializeInto(QwpTableBuffer tableBuffer, QwpTableBuffer.ColumnBuffer[] m default -> throw new LineSenderException("unknown staged row entry type: " + kind); } } - tableBuffer.nextRow(missingColumns, missingColumnCount); + tableBuffer.nextRow(rowFillColumns, rowFillColumnCount); } int size() { return size; } - private void appendEntry( + private void stageEntry( QwpTableBuffer.ColumnBuffer column, Object sidecarObject, int kind, From ea83e18aca5b550f165d170c2d8dea0ade6268e0 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 6 Mar 2026 15:53:26 +0100 Subject: [PATCH 165/230] ILP v4 -> QWP v1 --- .../main/java/io/questdb/client/Sender.java | 24 +++++++++---------- .../qwp/client/NativeBufferWriter.java | 2 +- .../cutlass/qwp/client/QwpBufferWriter.java | 4 ++-- .../qwp/client/QwpWebSocketEncoder.java | 2 +- .../qwp/client/QwpWebSocketSender.java | 8 +++---- .../cutlass/qwp/client/WebSocketResponse.java | 2 +- .../cutlass/qwp/protocol/QwpBitWriter.java | 2 +- .../cutlass/qwp/protocol/QwpColumnDef.java | 4 ++-- .../cutlass/qwp/protocol/QwpConstants.java | 2 +- .../qwp/protocol/QwpGorillaEncoder.java | 2 +- .../cutlass/qwp/protocol/QwpSchemaHash.java | 6 ++--- 11 files changed, 29 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 5c40de9..a58274c 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -474,7 +474,7 @@ enum Transport { /** * Use WebSocket transport to communicate with a QuestDB server. *

    - * WebSocket transport uses the ILP v4 binary protocol for efficient data ingestion. + * WebSocket transport uses the QWP v1 binary protocol for efficient data ingestion. * It supports both synchronous and asynchronous modes with flow control. */ WEBSOCKET @@ -1391,6 +1391,17 @@ private static RuntimeException rethrow(Throwable t) { throw new LineSenderException(t); } + private String buildWebSocketAuthHeader() { + if (username != null && password != null) { + String credentials = username + ":" + password; + return "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + } + if (httpToken != null) { + return "Bearer " + httpToken; + } + return null; + } + private void configureDefaults() { if (protocol == PARAMETER_NOT_SET_EXPLICITLY) { protocol = PROTOCOL_TCP; @@ -1797,17 +1808,6 @@ private void validateParameters() { } } - private String buildWebSocketAuthHeader() { - if (username != null && password != null) { - String credentials = username + ":" + password; - return "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); - } - if (httpToken != null) { - return "Bearer " + httpToken; - } - return null; - } - private void websocket() { if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol was already configured ") diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index e538769..4fe3262 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -29,7 +29,7 @@ import io.questdb.client.std.Unsafe; /** - * A simple native memory buffer writer for encoding ILP v4 messages. + * A simple native memory buffer writer for encoding QWP v1 messages. *

    * This class provides write methods similar to HttpClient.Request but writes * to a native memory buffer that can be sent over WebSocket. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java index 644fdf8..34347c6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java @@ -27,10 +27,10 @@ import io.questdb.client.cutlass.line.array.ArrayBufferAppender; /** - * Buffer writer interface for ILP v4 message encoding. + * Buffer writer interface for QWP v1 message encoding. *

    * This interface extends {@link ArrayBufferAppender} with additional methods - * required for encoding ILP v4 messages, including varint encoding, string + * required for encoding QWP v1 messages, including varint encoding, string * handling, and buffer manipulation. *

    * Implementations include: diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 632e4c8..28cea79 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -34,7 +34,7 @@ import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** - * Encodes ILP v4 messages for WebSocket transport. + * Encodes QWP v1 messages for WebSocket transport. *

    * This encoder reads column data from off-heap {@link io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory} * buffers in {@link QwpTableBuffer.ColumnBuffer} and uses bulk {@code putBlockOfBytes} for fixed-width diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 2cfb689..00aea36 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -53,7 +53,7 @@ /** - * ILP v4 WebSocket client sender for streaming data to QuestDB. + * QWP v1 WebSocket client sender for streaming data to QuestDB. *

    * This sender uses a double-buffering scheme with asynchronous I/O for high throughput: *

      @@ -123,7 +123,7 @@ public class QwpWebSocketSender implements Sender { // Auto-flush configuration private final int autoFlushRows; private final Decimal256 currentDecimal256 = new Decimal256(); - // Encoder for ILP v4 messages + // Encoder for QWP v1 messages private final QwpWebSocketEncoder encoder; // Global symbol dictionary for delta encoding private final GlobalSymbolDictionary globalSymbolDictionary; @@ -1088,7 +1088,7 @@ private void failExpectedIfNeeded(long expectedSequence, LineSenderException err /** * Flushes pending rows by encoding and sending them. - * Each table's rows are encoded into a separate ILP v4 message and sent as one WebSocket frame. + * Each table's rows are encoded into a separate QWP v1 message and sent as one WebSocket frame. */ private void flushPendingRows() { if (pendingRowCount <= 0) { @@ -1139,7 +1139,7 @@ private void flushPendingRows() { QwpBufferWriter buffer = encoder.getBuffer(); // Copy to microbatch buffer and seal immediately - // Each ILP v4 message must be in its own WebSocket frame + // Each QWP v1 message must be in its own WebSocket frame activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 0070a5e..f9f6c01 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -29,7 +29,7 @@ import java.nio.charset.StandardCharsets; /** - * Binary response format for WebSocket ILP v4 protocol. + * Binary response format for WebSocket QWP v1 protocol. *

      * Response format (little-endian): * 

      diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
index fbe6efd..c9ad763 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
@@ -28,7 +28,7 @@
 import io.questdb.client.std.Unsafe;
 
 /**
- * Bit-level writer for ILP v4 protocol.
+ * Bit-level writer for QWP v1 protocol.
  * 
      
  * This class writes bits to a buffer in LSB-first order within each byte.
  * Bits are packed sequentially, spanning byte boundaries as needed.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
index a2f4f50..c7355ac 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
@@ -28,7 +28,7 @@
 import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_CHAR;
 
 /**
- * Represents a column definition in an ILP v4 schema.
+ * Represents a column definition in an QWP v1 schema.
  * 
      
  * This class is immutable and safe for caching.
  */
@@ -41,7 +41,7 @@ public final class QwpColumnDef {
      * Creates a column definition.
      *
      * @param name     the column name (UTF-8)
-     * @param typeCode the ILP v4 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable)
+     * @param typeCode the QWP v1 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable)
      */
     public QwpColumnDef(String name, byte typeCode) {
         this.name = name;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
index 216f2fa..b605f2e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
@@ -25,7 +25,7 @@
 package io.questdb.client.cutlass.qwp.protocol;
 
 /**
- * Constants for the ILP v4 binary protocol.
+ * Constants for the QWP v1 binary protocol.
  */
 public final class QwpConstants {
 
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
index 302299c..5d1ed4f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
@@ -28,7 +28,7 @@
 import io.questdb.client.std.Unsafe;
 
 /**
- * Gorilla delta-of-delta encoder for timestamps in ILP v4 format.
+ * Gorilla delta-of-delta encoder for timestamps in QWP v1 format.
  * 
      
  * This encoder is used by the WebSocket encoder to compress timestamp columns.
  * It uses delta-of-delta compression where:
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
index 63d005d..bb806cc 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
@@ -28,7 +28,7 @@
 import io.questdb.client.std.Unsafe;
 
 /**
- * XXHash64 implementation for schema hashing in ILP v4 protocol.
+ * XXHash64 implementation for schema hashing in QWP v1 protocol.
  * 
      
  * The schema hash is computed over column definitions (name + type) to enable
  * schema caching. When a client sends a schema reference (hash), the server
@@ -41,7 +41,7 @@
  */
 public final class QwpSchemaHash {
 
-    // Default seed (0 for ILP v4)
+    // Default seed (0 for QWP v1)
     private static final long DEFAULT_SEED = 0L;
     // XXHash64 constants
     private static final long PRIME64_1 = 0x9E3779B185EBCA87L;
@@ -57,7 +57,7 @@ private QwpSchemaHash() {
     }
 
     /**
-     * Computes the schema hash for ILP v4 using String column names.
+     * Computes the schema hash for QWP v1 using String column names.
      * Note: Iterates over String chars and converts to UTF-8 bytes directly to avoid getBytes() allocation.
      *
      * @param columnNames array of column names

From 21330636ea95050cc15dc144775818b938091a29 Mon Sep 17 00:00:00 2001
From: Jaromir Hamala 
Date: Fri, 6 Mar 2026 16:43:07 +0100
Subject: [PATCH 166/230] better row rollback and optimize array staging

---
 .../io/questdb/client/cairo/ColumnType.java   |   3 +-
 .../cutlass/qwp/client/QwpUdpSender.java      | 490 +++++++++++-------
 .../qwp/protocol/OffHeapAppendMemory.java     |   3 +-
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |  80 +++
 .../cutlass/qwp/client/QwpUdpSenderTest.java  | 268 ++++++++++
 .../qwp/protocol/QwpTableBufferTest.java      |  64 +++
 6 files changed, 723 insertions(+), 185 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cairo/ColumnType.java b/core/src/main/java/io/questdb/client/cairo/ColumnType.java
index 275650b..89d4af3 100644
--- a/core/src/main/java/io/questdb/client/cairo/ColumnType.java
+++ b/core/src/main/java/io/questdb/client/cairo/ColumnType.java
@@ -352,6 +352,7 @@ private static int mkGeoHashType(int bits, short baseType) {
         typeNameMap.put(NULL, "NULL");
 
         arrayTypeSet.add(DOUBLE);
+        arrayTypeSet.add(LONG);
 
         TYPE_SIZE_POW2[UNDEFINED] = -1;
         TYPE_SIZE_POW2[BOOLEAN] = 0;
@@ -504,4 +505,4 @@ private static int mkGeoHashType(int bits, short baseType) {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
index 56fa21c..1ec6c8b 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
@@ -26,7 +26,6 @@
 
 import io.questdb.client.Sender;
 import io.questdb.client.cutlass.line.LineSenderException;
-import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
 import io.questdb.client.cutlass.line.array.DoubleArray;
 import io.questdb.client.cutlass.line.array.LongArray;
 import io.questdb.client.cutlass.line.udp.UdpLineChannel;
@@ -82,7 +81,6 @@ public class QwpUdpSender implements Sender {
     private static final int SAFETY_MARGIN_BYTES = 8;
     private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class);
 
-    private final ArraySizeCounter arraySizeCounter = new ArraySizeCounter();
     private final UdpLineChannel channel;
     private final QwpColumnWriter columnWriter = new QwpColumnWriter();
     private final NativeBufferWriter headerBuffer = new NativeBufferWriter();
@@ -140,14 +138,24 @@ public void at(Instant timestamp) {
     public void atNow() {
         checkNotClosed();
         checkTableSelected();
-        commitCurrentRow();
+        try {
+            commitCurrentRow();
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
     }
 
     @Override
     public Sender boolColumn(CharSequence columnName, boolean value) {
         checkNotClosed();
         checkTableSelected();
-        stageBooleanColumnValue(columnName, value);
+        try {
+            stageBooleanColumnValue(columnName, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -159,13 +167,7 @@ public DirectByteSlice bufferView() {
     @Override
     public void cancelRow() {
         checkNotClosed();
-        if (currentTableBuffer != null) {
-            currentTableBuffer.cancelCurrentRow();
-            currentTableBuffer.rollbackUncommittedColumns();
-        }
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
-        clearStagedRow();
+        rollbackCurrentRowToCommittedState();
     }
 
     @Override
@@ -173,11 +175,7 @@ public void close() {
         if (!closed) {
             try {
                 if (hasInProgressRow()) {
-                    currentTableBuffer.cancelCurrentRow();
-                    currentTableBuffer.rollbackUncommittedColumns();
-                    cachedTimestampColumn = null;
-                    cachedTimestampNanosColumn = null;
-                    clearStagedRow();
+                    rollbackCurrentRowToCommittedState();
                 }
                 flushInternal();
             } catch (Exception e) {
@@ -210,7 +208,12 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageDecimal64ColumnValue(name, value);
+        try {
+            stageDecimal64ColumnValue(name, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -221,7 +224,12 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageDecimal128ColumnValue(name, value);
+        try {
+            stageDecimal128ColumnValue(name, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -232,7 +240,12 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageDecimal256ColumnValue(name, value);
+        try {
+            stageDecimal256ColumnValue(name, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -243,7 +256,12 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageDoubleArrayColumnValue(name, values);
+        try {
+            stageDoubleArrayColumnValue(name, values);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -254,7 +272,12 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageDoubleArrayColumnValue(name, values);
+        try {
+            stageDoubleArrayColumnValue(name, values);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -265,7 +288,12 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageDoubleArrayColumnValue(name, values);
+        try {
+            stageDoubleArrayColumnValue(name, values);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -276,7 +304,12 @@ public Sender doubleArray(CharSequence name, DoubleArray array) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageDoubleArrayColumnValue(name, array);
+        try {
+            stageDoubleArrayColumnValue(name, array);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -284,7 +317,12 @@ public Sender doubleArray(CharSequence name, DoubleArray array) {
     public Sender doubleColumn(CharSequence columnName, double value) {
         checkNotClosed();
         checkTableSelected();
-        stageDoubleColumnValue(columnName, value);
+        try {
+            stageDoubleColumnValue(columnName, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -302,7 +340,12 @@ public Sender longArray(@NotNull CharSequence name, long[] values) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageLongArrayColumnValue(name, values);
+        try {
+            stageLongArrayColumnValue(name, values);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -313,7 +356,12 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageLongArrayColumnValue(name, values);
+        try {
+            stageLongArrayColumnValue(name, values);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -324,7 +372,12 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageLongArrayColumnValue(name, values);
+        try {
+            stageLongArrayColumnValue(name, values);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -335,7 +388,12 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) {
         }
         checkNotClosed();
         checkTableSelected();
-        stageLongArrayColumnValue(name, array);
+        try {
+            stageLongArrayColumnValue(name, array);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -343,7 +401,12 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) {
     public Sender longColumn(CharSequence columnName, long value) {
         checkNotClosed();
         checkTableSelected();
-        stageLongColumnValue(columnName, value);
+        try {
+            stageLongColumnValue(columnName, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -370,7 +433,12 @@ public void reset() {
     public Sender stringColumn(CharSequence columnName, CharSequence value) {
         checkNotClosed();
         checkTableSelected();
-        stageStringColumnValue(columnName, value);
+        try {
+            stageStringColumnValue(columnName, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -378,7 +446,12 @@ public Sender stringColumn(CharSequence columnName, CharSequence value) {
     public Sender symbol(CharSequence columnName, CharSequence value) {
         checkNotClosed();
         checkTableSelected();
-        stageSymbolColumnValue(columnName, value);
+        try {
+            stageSymbolColumnValue(columnName, value);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -410,11 +483,16 @@ public Sender table(CharSequence tableName) {
     public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) {
         checkNotClosed();
         checkTableSelected();
-        if (unit == ChronoUnit.NANOS) {
-            stageTimestampColumnValue(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS);
-        } else {
-            long micros = toMicros(value, unit);
-            stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS);
+        try {
+            if (unit == ChronoUnit.NANOS) {
+                stageTimestampColumnValue(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS);
+            } else {
+                long micros = toMicros(value, unit);
+                stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS);
+            }
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
         }
         return this;
     }
@@ -423,8 +501,13 @@ public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit un
     public Sender timestampColumn(CharSequence columnName, Instant value) {
         checkNotClosed();
         checkTableSelected();
-        long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L;
-        stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS);
+        try {
+            long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L;
+            stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS);
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
         return this;
     }
 
@@ -487,9 +570,9 @@ private void stageDesignatedTimestampValue(long value, boolean nanos) {
 
     private void stageDoubleArrayColumnValue(CharSequence name, Object value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true);
+        stagedRow.stageDoubleArray(col, value);
         stagedRowValueCount++;
         addStagedColumn(col);
-        stagedRow.stageDoubleArray(col, value, estimateDoubleArrayPayload(value));
     }
 
     private void stageDoubleColumnValue(CharSequence name, double value) {
@@ -501,9 +584,9 @@ private void stageDoubleColumnValue(CharSequence name, double value) {
 
     private void stageLongArrayColumnValue(CharSequence name, Object value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true);
+        stagedRow.stageLongArray(col, value);
         stagedRowValueCount++;
         addStagedColumn(col);
-        stagedRow.stageLongArray(col, value, estimateLongArrayPayload(value));
     }
 
     private void stageLongColumnValue(CharSequence name, long value) {
@@ -538,16 +621,26 @@ private void atMicros(long timestampMicros) {
         if (stagedRowValueCount == 0) {
             throw new LineSenderException("no columns were provided");
         }
-        stageDesignatedTimestampValue(timestampMicros, false);
-        commitCurrentRow();
+        try {
+            stageDesignatedTimestampValue(timestampMicros, false);
+            commitCurrentRow();
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
     }
 
     private void atNanos(long timestampNanos) {
         if (stagedRowValueCount == 0) {
             throw new LineSenderException("no columns were provided");
         }
-        stageDesignatedTimestampValue(timestampNanos, true);
-        commitCurrentRow();
+        try {
+            stageDesignatedTimestampValue(timestampNanos, true);
+            commitCurrentRow();
+        } catch (RuntimeException | Error e) {
+            rollbackCurrentRowToCommittedState();
+            throw e;
+        }
     }
 
     private void checkNotClosed() {
@@ -569,6 +662,16 @@ private void clearStagedRow() {
         rowFillColumnCount = 0;
     }
 
+    private void rollbackCurrentRowToCommittedState() {
+        if (currentTableBuffer != null) {
+            currentTableBuffer.cancelCurrentRow();
+            currentTableBuffer.rollbackUncommittedColumns();
+        }
+        cachedTimestampColumn = null;
+        cachedTimestampNanosColumn = null;
+        clearStagedRow();
+    }
+
     private void collectRowFillColumns(int targetRows) {
         rowFillColumnCount = 0;
         for (int i = 0, n = currentTableBuffer.getColumnCount(); i < n; i++) {
@@ -632,22 +735,6 @@ private int encodeTablePayloadForUdp(QwpTableBuffer tableBuffer) {
         return payloadWriter.getPosition();
     }
 
-    private long estimateArrayValueSize(int nDims, long elementCount) {
-        return 1L + (long) nDims * 4 + elementCount * 8;
-    }
-
-    private long estimateArrayValueSize(DoubleArray array) {
-        arraySizeCounter.reset();
-        array.appendToBufPtr(arraySizeCounter);
-        return arraySizeCounter.size;
-    }
-
-    private long estimateArrayValueSize(LongArray array) {
-        arraySizeCounter.reset();
-        array.appendToBufPtr(arraySizeCounter);
-        return arraySizeCounter.size;
-    }
-
     private long estimateBaseForCurrentSchema() {
         long estimate = HEADER_SIZE;
         int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName);
@@ -712,48 +799,6 @@ private long estimateStagedEntryPayload(int entryIndex, QwpTableBuffer.ColumnBuf
         };
     }
 
-    private long estimateDoubleArrayPayload(Object value) {
-        if (value instanceof double[] values) {
-            return estimateArrayValueSize(1, values.length);
-        }
-        if (value instanceof double[][] values) {
-            int dim0 = values.length;
-            int dim1 = dim0 > 0 ? values[0].length : 0;
-            return estimateArrayValueSize(2, (long) dim0 * dim1);
-        }
-        if (value instanceof double[][][] values) {
-            int dim0 = values.length;
-            int dim1 = dim0 > 0 ? values[0].length : 0;
-            int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0;
-            return estimateArrayValueSize(3, (long) dim0 * dim1 * dim2);
-        }
-        if (value instanceof DoubleArray values) {
-            return estimateArrayValueSize(values);
-        }
-        throw new LineSenderException("unsupported double array type");
-    }
-
-    private long estimateLongArrayPayload(Object value) {
-        if (value instanceof long[] values) {
-            return estimateArrayValueSize(1, values.length);
-        }
-        if (value instanceof long[][] values) {
-            int dim0 = values.length;
-            int dim1 = dim0 > 0 ? values[0].length : 0;
-            return estimateArrayValueSize(2, (long) dim0 * dim1);
-        }
-        if (value instanceof long[][][] values) {
-            int dim0 = values.length;
-            int dim1 = dim0 > 0 ? values[0].length : 0;
-            int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0;
-            return estimateArrayValueSize(3, (long) dim0 * dim1 * dim2);
-        }
-        if (value instanceof LongArray values) {
-            return estimateArrayValueSize(values);
-        }
-        throw new LineSenderException("unsupported long array type");
-    }
-
     private void ensureRowFillColumnCapacity(int required) {
         if (required <= rowFillColumns.length) {
             return;
@@ -875,34 +920,6 @@ private void commitStagedRow() {
         stagedRow.commitIntoTableBuffer(currentTableBuffer, rowFillColumns, rowFillColumnCount);
     }
 
-    private static void commitDoubleArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) {
-        if (value instanceof double[] values) {
-            col.addDoubleArray(values);
-        } else if (value instanceof double[][] values) {
-            col.addDoubleArray(values);
-        } else if (value instanceof double[][][] values) {
-            col.addDoubleArray(values);
-        } else if (value instanceof DoubleArray values) {
-            col.addDoubleArray(values);
-        } else {
-            throw new LineSenderException("unsupported double array type");
-        }
-    }
-
-    private static void commitLongArrayValue(QwpTableBuffer.ColumnBuffer col, Object value) {
-        if (value instanceof long[] values) {
-            col.addLongArray(values);
-        } else if (value instanceof long[][] values) {
-            col.addLongArray(values);
-        } else if (value instanceof long[][][] values) {
-            col.addLongArray(values);
-        } else if (value instanceof LongArray values) {
-            col.addLongArray(values);
-        } else {
-            throw new LineSenderException("unsupported long array type");
-        }
-    }
-
     private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) {
         return switch (type) {
             case TYPE_BOOLEAN -> packedBytes(valuesBefore + missing) - packedBytes(valuesBefore);
@@ -1005,39 +1022,6 @@ private static int utf8Length(CharSequence s) {
         return len;
     }
 
-    private static final class ArraySizeCounter implements ArrayBufferAppender {
-        private long size;
-
-        @Override
-        public void putBlockOfBytes(long from, long len) {
-            size += len;
-        }
-
-        @Override
-        public void putByte(byte b) {
-            size++;
-        }
-
-        @Override
-        public void putDouble(double value) {
-            size += 8;
-        }
-
-        @Override
-        public void putInt(int value) {
-            size += 4;
-        }
-
-        @Override
-        public void putLong(long value) {
-            size += 8;
-        }
-
-        private void reset() {
-            size = 0;
-        }
-    }
-
     private static final class NativeRowStaging {
         private static final int AUX_INT_OFFSET = 4;
         private static final int ENTRY_SIZE = 40;
@@ -1053,20 +1037,18 @@ private static final class NativeRowStaging {
         private final OffHeapAppendMemory varData = new OffHeapAppendMemory(128);
         private QwpTableBuffer.ColumnBuffer[] columns = new QwpTableBuffer.ColumnBuffer[8];
         private int size;
-        private Object[] sidecarObjects = new Object[8];
 
         void stageBoolean(QwpTableBuffer.ColumnBuffer column, boolean value) {
-            stageEntry(column, null, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0);
+            stageEntry(column, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0);
         }
 
         void stageDecimal128(QwpTableBuffer.ColumnBuffer column, Decimal128 value) {
-            stageEntry(column, null, ENTRY_DECIMAL128, value.getScale(), value.getHigh(), value.getLow(), 0, 0);
+            stageEntry(column, ENTRY_DECIMAL128, value.getScale(), value.getHigh(), value.getLow(), 0, 0);
         }
 
         void stageDecimal256(QwpTableBuffer.ColumnBuffer column, Decimal256 value) {
             stageEntry(
                     column,
-                    null,
                     ENTRY_DECIMAL256,
                     value.getScale(),
                     value.getHh(),
@@ -1077,23 +1059,39 @@ void stageDecimal256(QwpTableBuffer.ColumnBuffer column, Decimal256 value) {
         }
 
         void stageDecimal64(QwpTableBuffer.ColumnBuffer column, Decimal64 value) {
-            stageEntry(column, null, ENTRY_DECIMAL64, value.getScale(), value.getValue(), 0, 0, 0);
+            stageEntry(column, ENTRY_DECIMAL64, value.getScale(), value.getValue(), 0, 0, 0);
         }
 
         void stageDouble(QwpTableBuffer.ColumnBuffer column, double value) {
-            stageEntry(column, null, ENTRY_DOUBLE, 0, Double.doubleToRawLongBits(value), 0, 0, 0);
+            stageEntry(column, ENTRY_DOUBLE, 0, Double.doubleToRawLongBits(value), 0, 0, 0);
         }
 
-        void stageDoubleArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) {
-            stageEntry(column, value, ENTRY_DOUBLE_ARRAY, 0, estimatePayload, 0, 0, 0);
+        void stageDoubleArray(QwpTableBuffer.ColumnBuffer column, Object value) {
+            long offset = varData.getAppendOffset();
+            try {
+                appendDoubleArrayPayload(value);
+                long payloadLength = varData.getAppendOffset() - offset;
+                stageEntry(column, ENTRY_DOUBLE_ARRAY, 0, payloadLength, offset, 0, 0);
+            } catch (Throwable t) {
+                varData.jumpTo(offset);
+                throw t;
+            }
         }
 
         void stageLong(QwpTableBuffer.ColumnBuffer column, int kind, long value) {
-            stageEntry(column, null, kind, 0, value, 0, 0, 0);
+            stageEntry(column, kind, 0, value, 0, 0, 0);
         }
 
-        void stageLongArray(QwpTableBuffer.ColumnBuffer column, Object value, long estimatePayload) {
-            stageEntry(column, value, ENTRY_LONG_ARRAY, 0, estimatePayload, 0, 0, 0);
+        void stageLongArray(QwpTableBuffer.ColumnBuffer column, Object value) {
+            long offset = varData.getAppendOffset();
+            try {
+                appendLongArrayPayload(value);
+                long payloadLength = varData.getAppendOffset() - offset;
+                stageEntry(column, ENTRY_LONG_ARRAY, 0, payloadLength, offset, 0, 0);
+            } catch (Throwable t) {
+                varData.jumpTo(offset);
+                throw t;
+            }
         }
 
         void stageUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value, long long1) {
@@ -1104,13 +1102,12 @@ void stageUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value,
                 varData.putUtf8(value);
                 len = (int) (varData.getAppendOffset() - offset);
             }
-            stageEntry(column, null, kind, len, offset, long1, 0, 0);
+            stageEntry(column, kind, len, offset, long1, 0, 0);
         }
 
         void clear() {
             for (int i = 0; i < size; i++) {
                 columns[i] = null;
-                sidecarObjects[i] = null;
             }
             size = 0;
             entries.truncate();
@@ -1175,8 +1172,8 @@ void commitIntoTableBuffer(QwpTableBuffer tableBuffer, QwpTableBuffer.ColumnBuff
                         column.addDecimal256(decimal256Sink);
                     }
                     case ENTRY_DOUBLE -> column.addDouble(Double.longBitsToDouble(long0));
-                    case ENTRY_DOUBLE_ARRAY -> commitDoubleArrayValue(column, sidecarObjects[i]);
-                    case ENTRY_LONG_ARRAY -> commitLongArrayValue(column, sidecarObjects[i]);
+                    case ENTRY_DOUBLE_ARRAY -> column.addDoubleArrayPayload(varData.addressOf(long1), long0);
+                    case ENTRY_LONG_ARRAY -> column.addLongArrayPayload(varData.addressOf(long1), long0);
                     case ENTRY_STRING -> column.addStringUtf8(varData.addressOf(long0), auxInt);
                     case ENTRY_SYMBOL -> {
                         if (auxInt < 0) {
@@ -1197,7 +1194,6 @@ int size() {
 
         private void stageEntry(
                 QwpTableBuffer.ColumnBuffer column,
-                Object sidecarObject,
                 int kind,
                 int auxInt,
                 long long0,
@@ -1207,7 +1203,6 @@ private void stageEntry(
         ) {
             ensureCapacity(size + 1);
             columns[size] = column;
-            sidecarObjects[size] = sidecarObject;
             entries.putInt(kind);
             entries.putInt(auxInt);
             entries.putLong(long0);
@@ -1234,10 +1229,139 @@ private void ensureCapacity(int required) {
             QwpTableBuffer.ColumnBuffer[] newColumns = new QwpTableBuffer.ColumnBuffer[newCapacity];
             System.arraycopy(columns, 0, newColumns, 0, size);
             columns = newColumns;
+        }
+
+        private static int checkedArrayElementCount(long product) {
+            if (product > Integer.MAX_VALUE) {
+                throw new LineSenderException("array too large: total element count exceeds int range");
+            }
+            return (int) product;
+        }
 
-            Object[] newSidecarObjects = new Object[newCapacity];
-            System.arraycopy(sidecarObjects, 0, newSidecarObjects, 0, size);
-            sidecarObjects = newSidecarObjects;
+        private void appendDoubleArrayPayload(Object value) {
+            if (value instanceof double[] values) {
+                varData.putByte((byte) 1);
+                varData.putInt(values.length);
+                for (double v : values) {
+                    varData.putDouble(v);
+                }
+                return;
+            }
+            if (value instanceof double[][] values) {
+                int dim0 = values.length;
+                int dim1 = dim0 > 0 ? values[0].length : 0;
+                for (int i = 1; i < dim0; i++) {
+                    if (values[i].length != dim1) {
+                        throw new LineSenderException("irregular array shape");
+                    }
+                }
+                checkedArrayElementCount((long) dim0 * dim1);
+                varData.putByte((byte) 2);
+                varData.putInt(dim0);
+                varData.putInt(dim1);
+                for (double[] row : values) {
+                    for (double v : row) {
+                        varData.putDouble(v);
+                    }
+                }
+                return;
+            }
+            if (value instanceof double[][][] values) {
+                int dim0 = values.length;
+                int dim1 = dim0 > 0 ? values[0].length : 0;
+                int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0;
+                for (int i = 0; i < dim0; i++) {
+                    if (values[i].length != dim1) {
+                        throw new LineSenderException("irregular array shape");
+                    }
+                    for (int j = 0; j < dim1; j++) {
+                        if (values[i][j].length != dim2) {
+                            throw new LineSenderException("irregular array shape");
+                        }
+                    }
+                }
+                checkedArrayElementCount((long) dim0 * dim1 * dim2);
+                varData.putByte((byte) 3);
+                varData.putInt(dim0);
+                varData.putInt(dim1);
+                varData.putInt(dim2);
+                for (double[][] plane : values) {
+                    for (double[] row : plane) {
+                        for (double v : row) {
+                            varData.putDouble(v);
+                        }
+                    }
+                }
+                return;
+            }
+            if (value instanceof DoubleArray values) {
+                values.appendToBufPtr(varData);
+                return;
+            }
+            throw new LineSenderException("unsupported double array type");
+        }
+
+        private void appendLongArrayPayload(Object value) {
+            if (value instanceof long[] values) {
+                varData.putByte((byte) 1);
+                varData.putInt(values.length);
+                for (long v : values) {
+                    varData.putLong(v);
+                }
+                return;
+            }
+            if (value instanceof long[][] values) {
+                int dim0 = values.length;
+                int dim1 = dim0 > 0 ? values[0].length : 0;
+                for (int i = 1; i < dim0; i++) {
+                    if (values[i].length != dim1) {
+                        throw new LineSenderException("irregular array shape");
+                    }
+                }
+                checkedArrayElementCount((long) dim0 * dim1);
+                varData.putByte((byte) 2);
+                varData.putInt(dim0);
+                varData.putInt(dim1);
+                for (long[] row : values) {
+                    for (long v : row) {
+                        varData.putLong(v);
+                    }
+                }
+                return;
+            }
+            if (value instanceof long[][][] values) {
+                int dim0 = values.length;
+                int dim1 = dim0 > 0 ? values[0].length : 0;
+                int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0;
+                for (int i = 0; i < dim0; i++) {
+                    if (values[i].length != dim1) {
+                        throw new LineSenderException("irregular array shape");
+                    }
+                    for (int j = 0; j < dim1; j++) {
+                        if (values[i][j].length != dim2) {
+                            throw new LineSenderException("irregular array shape");
+                        }
+                    }
+                }
+                checkedArrayElementCount((long) dim0 * dim1 * dim2);
+                varData.putByte((byte) 3);
+                varData.putInt(dim0);
+                varData.putInt(dim1);
+                varData.putInt(dim2);
+                for (long[][] plane : values) {
+                    for (long[] row : plane) {
+                        for (long v : row) {
+                            varData.putLong(v);
+                        }
+                    }
+                }
+                return;
+            }
+            if (value instanceof LongArray values) {
+                values.appendToBufPtr(varData);
+                return;
+            }
+            throw new LineSenderException("unsupported long array type");
         }
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
index 6d92e57..308a723 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.protocol;
 
+import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
 import io.questdb.client.std.Unsafe;
@@ -38,7 +39,7 @@
  * 
      
  * Growth strategy: capacity doubles on each resize via {@link Unsafe#realloc}.
  */
-public class OffHeapAppendMemory implements QuietCloseable {
+public class OffHeapAppendMemory implements ArrayBufferAppender, QuietCloseable {
 
     private static final int DEFAULT_INITIAL_CAPACITY = 128;
     private long appendAddress;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index 9bddd9b..55090eb 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.protocol;
 
+import io.questdb.client.cairo.ColumnType;
 import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
 import io.questdb.client.cutlass.line.array.DoubleArray;
@@ -487,6 +488,8 @@ void reset(boolean forLong) {
      * operation and efficient bulk copy to network buffers.
      */
     public static class ColumnBuffer implements QuietCloseable {
+        private static final long DOUBLE_ARRAY_BASE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(double[].class);
+        private static final long LONG_ARRAY_BASE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(long[].class);
         final int elemSize;
         final String name;
         final boolean nullable;
@@ -751,6 +754,10 @@ public void addDoubleArray(DoubleArray array) {
             size++;
         }
 
+        public void addDoubleArrayPayload(long ptr, long len) {
+            appendArrayPayload(ptr, len, false);
+        }
+
         public void addFloat(float value) {
             ensureNullBitmapForNonNull();
             dataBuffer.putFloat(value);
@@ -901,6 +908,10 @@ public void addLongArray(LongArray array) {
             size++;
         }
 
+        public void addLongArrayPayload(long ptr, long len) {
+            appendArrayPayload(ptr, len, true);
+        }
+
         public void addNull() {
             if (nullable) {
                 ensureNullCapacity(size + 1);
@@ -1326,6 +1337,75 @@ private static int checkedElementCount(long product) {
             return (int) product;
         }
 
+        private void appendArrayPayload(long ptr, long len, boolean forLong) {
+            if (len < 0) {
+                addNull();
+                return;
+            }
+            if (len == 0) {
+                throw new LineSenderException("invalid array payload: empty payload");
+            }
+
+            int nDims = Unsafe.getUnsafe().getByte(ptr) & 0xFF;
+            if (nDims < 1 || nDims > ColumnType.ARRAY_NDIMS_LIMIT) {
+                throw new LineSenderException("invalid array payload: bad dimensionality " + nDims);
+            }
+
+            long cursor = ptr + 1;
+            long headerBytes = 1L + (long) nDims * Integer.BYTES;
+            if (len < headerBytes) {
+                throw new LineSenderException("invalid array payload: truncated shape header");
+            }
+
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                int dimLen = Unsafe.getUnsafe().getInt(cursor);
+                if (dimLen < 0) {
+                    throw new LineSenderException("invalid array payload: negative dimension length");
+                }
+                elemCount = checkedElementCount((long) elemCount * dimLen);
+                cursor += Integer.BYTES;
+            }
+
+            long dataBytes = (long) elemCount * (forLong ? Long.BYTES : Double.BYTES);
+            if (len != headerBytes + dataBytes) {
+                throw new LineSenderException("invalid array payload: length mismatch");
+            }
+
+            ensureArrayCapacity(nDims, elemCount);
+            arrayDims[valueCount] = (byte) nDims;
+
+            cursor = ptr + 1;
+            for (int d = 0; d < nDims; d++) {
+                arrayShapes[arrayShapeOffset++] = Unsafe.getUnsafe().getInt(cursor);
+                cursor += Integer.BYTES;
+            }
+
+            if (dataBytes > 0) {
+                if (forLong) {
+                    Unsafe.getUnsafe().copyMemory(
+                            null,
+                            cursor,
+                            longArrayData,
+                            LONG_ARRAY_BASE_OFFSET + (long) arrayDataOffset * Long.BYTES,
+                            dataBytes
+                    );
+                } else {
+                    Unsafe.getUnsafe().copyMemory(
+                            null,
+                            cursor,
+                            doubleArrayData,
+                            DOUBLE_ARRAY_BASE_OFFSET + (long) arrayDataOffset * Double.BYTES,
+                            dataBytes
+                    );
+                }
+            }
+
+            arrayDataOffset += elemCount;
+            valueCount++;
+            size++;
+        }
+
         private void allocateStorage(byte type) {
             switch (type) {
                 case TYPE_BOOLEAN:
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
index f2bf425..0b9bc73 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
@@ -25,6 +25,8 @@
 package io.questdb.client.test.cutlass.qwp.client;
 
 import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.line.array.DoubleArray;
+import io.questdb.client.cutlass.line.array.LongArray;
 import io.questdb.client.cutlass.qwp.client.QwpUdpSender;
 import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
 import io.questdb.client.network.NetworkFacadeImpl;
@@ -265,6 +267,114 @@ public void testBoundedSenderArrayReplayPreservesRowsAndPacketLimit() throws Exc
         });
     }
 
+    @Test
+    public void testBoundedSenderHigherDimensionalDoubleArrayWrapperPreservesRowsAndPacketLimit() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("arrays", sender -> {
+                                try (DoubleArray doubleArray = new DoubleArray(2, 1, 1, 2)) {
+                                    doubleArray.append(1.25).append(2.25).append(3.25).append(4.25);
+                                    sender.table("arrays")
+                                            .symbol("sym", "alpha")
+                                            .longArray("la", new long[][][]{{{1, 2}}, {{3, 4}}})
+                                            .doubleArray("da", doubleArray)
+                                            .atNow();
+                                }
+                            },
+                            "sym", "alpha",
+                            "la", longArrayValue(shape(2, 1, 2), 1, 2, 3, 4),
+                            "da", doubleArrayValue(shape(2, 1, 1, 2), 1.25, 2.25, 3.25, 4.25)),
+                    row("arrays", sender -> {
+                                try (DoubleArray doubleArray = new DoubleArray(1, 2, 1, 2)) {
+                                    doubleArray.append(10.5).append(20.5).append(30.5).append(40.5);
+                                    sender.table("arrays")
+                                            .symbol("sym", "beta")
+                                            .longArray("la", new long[][]{{10, 20}, {30, 40}})
+                                            .doubleArray("da", doubleArray)
+                                            .atNow();
+                                }
+                            },
+                            "sym", "beta",
+                            "la", longArrayValue(shape(2, 2), 10, 20, 30, 40),
+                            "da", doubleArrayValue(shape(1, 2, 1, 2), 10.5, 20.5, 30.5, 40.5))
+            );
+
+            int maxDatagramSize = fullPacketSize(rows) - 1;
+            RunResult result = runScenario(rows, maxDatagramSize);
+
+            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
+            assertPacketsWithinLimit(result, maxDatagramSize);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
+    @Test
+    public void testArrayWrapperStagingSnapshotsMutationAndCancelRow() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024);
+                 DoubleArray doubleArray = new DoubleArray(2)) {
+                sender.table("arrays");
+                long[] longValues = {1, 2};
+
+                doubleArray.append(1.5).append(2.5);
+                sender.longArray("la", longValues);
+                sender.doubleArray("da", doubleArray);
+
+                longValues[0] = 9;
+                longValues[1] = 10;
+                doubleArray.clear();
+                doubleArray.append(9.5).append(10.5);
+                sender.cancelRow();
+
+                sender.longArray("la", longValues);
+                sender.doubleArray("da", doubleArray);
+                longValues[0] = 100;
+                longValues[1] = 200;
+                doubleArray.clear();
+                doubleArray.append(100.5).append(200.5);
+                sender.atNow();
+                sender.flush();
+            }
+
+            assertRowsEqual(
+                    Arrays.asList(decodedRow(
+                            "arrays",
+                            "la", longArrayValue(shape(2), 9, 10),
+                            "da", doubleArrayValue(shape(2), 9.5, 10.5)
+                    )),
+                    decodeRows(nf.packets)
+            );
+        });
+    }
+
+    @Test
+    public void testLongArrayWrapperStagingSnapshotsMutation() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024);
+                 LongArray longArray = new LongArray(2, 2)) {
+                sender.table("arrays");
+
+                longArray.append(1).append(2).append(3).append(4);
+                sender.longArray("la", longArray);
+
+                longArray.clear();
+                longArray.append(10).append(20).append(30).append(40);
+                sender.atNow();
+                sender.flush();
+            }
+
+            assertRowsEqual(
+                    Arrays.asList(decodedRow(
+                            "arrays",
+                            "la", longArrayValue(shape(2, 2), 1, 2, 3, 4)
+                    )),
+                    decodeRows(nf.packets)
+            );
+        });
+    }
+
     @Test
     public void testBoundedSenderMixedTypesPreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
@@ -566,6 +676,45 @@ public void testOversizedArrayRowRejectedUsesActualEncodedSize() throws Exceptio
         });
     }
 
+    @Test
+    public void testIrregularArrayRejectedDuringStagingAndSenderRemainsUsable() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("arrays");
+                assertThrowsContains("irregular array shape", () ->
+                        sender.doubleArray("da", new double[][]{{1.0}, {2.0, 3.0}})
+                );
+
+                sender.table("ok");
+                sender.longColumn("x", 1).atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(1, nf.sendCount);
+            assertRowsEqual(Arrays.asList(decodedRow("ok", "x", 1L)), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testIrregularArrayRejectedDuringStagingDoesNotLeakColumnIntoSameTable() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("arrays");
+                assertThrowsContains("irregular array shape", () ->
+                        sender.doubleArray("bad", new double[][]{{1.0}, {2.0, 3.0}})
+                );
+
+                sender.longColumn("x", 1).atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(1, nf.sendCount);
+            assertRowsEqual(Arrays.asList(decodedRow("arrays", "x", 1L)), decodeRows(nf.packets));
+        });
+    }
+
     @Test
     public void testSchemaChangeMidRowFlushesImmediatelyAndPreservesRows() throws Exception {
         assertMemoryLeak(() -> {
@@ -767,6 +916,125 @@ public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() thr
         });
     }
 
+    @Test
+    public void testAtNowOversizeFailureRollsBackWithoutExplicitCancel() throws Exception {
+        assertMemoryLeak(() -> {
+            String large = repeat('x', 5000);
+            List oversizedRow = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("a", 2)
+                                    .stringColumn("s", large)
+                                    .atNow(),
+                            "a", 2L,
+                            "s", large)
+            );
+            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
+
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .atNow();
+
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("a", 2)
+                                .stringColumn("s", large)
+                                .atNow()
+                );
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.longColumn("a", 3).atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L),
+                    decodedRow("t", "a", 3L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testAtMicrosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
+        assertMemoryLeak(() -> {
+            String large = repeat('x', 5000);
+            List oversizedRow = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("a", 2)
+                                    .stringColumn("s", large)
+                                    .at(2_000_000L, ChronoUnit.MICROS),
+                            "a", 2L,
+                            "s", large,
+                            "", 2_000_000L)
+            );
+            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
+
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .at(1_000_000L, ChronoUnit.MICROS);
+
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("a", 2)
+                                .stringColumn("s", large)
+                                .at(2_000_000L, ChronoUnit.MICROS)
+                );
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.MICROS);
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L, "", 1_000_000L),
+                    decodedRow("t", "a", 3L, "", 3_000_000L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testAtNanosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
+        assertMemoryLeak(() -> {
+            String large = repeat('x', 5000);
+            List oversizedRow = Arrays.asList(
+                    row("tn", sender -> sender.table("tn")
+                                    .longColumn("a", 2)
+                                    .stringColumn("s", large)
+                                    .at(2_000_000L, ChronoUnit.NANOS),
+                            "a", 2L,
+                            "s", large,
+                            "", 2_000_000L)
+            );
+            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
+
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("tn")
+                        .longColumn("a", 1)
+                        .at(1_000_000L, ChronoUnit.NANOS);
+
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("a", 2)
+                                .stringColumn("s", large)
+                                .at(2_000_000L, ChronoUnit.NANOS)
+                );
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.NANOS);
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("tn", "a", 1L, "", 1_000_000L),
+                    decodedRow("tn", "a", 3L, "", 3_000_000L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
     @Test
     public void testSchemaChangeWithCommittedRowsFlushesImmediately() throws Exception {
         assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
index a04e0b5..eead658 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
@@ -26,6 +26,8 @@
 
 import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.line.array.DoubleArray;
+import io.questdb.client.cutlass.line.array.LongArray;
+import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
 import io.questdb.client.std.Unsafe;
@@ -511,6 +513,35 @@ public void testCancelRowRewindsMultiDimArrayOffsets() throws Exception {
         });
     }
 
+    @Test
+    public void testAddDoubleArrayPayloadSupportsHigherDimensionalShape() throws Exception {
+        assertMemoryLeak(() -> {
+            try (QwpTableBuffer table = new QwpTableBuffer("test");
+                 DoubleArray array = new DoubleArray(2, 1, 1, 2);
+                 OffHeapAppendMemory payload = new OffHeapAppendMemory(128)) {
+                QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+
+                array.append(1.0).append(2.0).append(3.0).append(4.0);
+                array.appendToBufPtr(payload);
+
+                col.addDoubleArrayPayload(payload.pageAddress(), payload.getAppendOffset());
+                table.nextRow();
+
+                assertEquals(1, col.getValueCount());
+
+                byte[] dims = col.getArrayDims();
+                int[] shapes = col.getArrayShapes();
+                assertEquals(4, dims[0]);
+                assertEquals(2, shapes[0]);
+                assertEquals(1, shapes[1]);
+                assertEquals(1, shapes[2]);
+                assertEquals(2, shapes[3]);
+
+                assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, readDoubleArraysLikeEncoder(col), 0.0);
+            }
+        });
+    }
+
     @Test
     public void testDoubleArrayWrapperMultipleRows() throws Exception {
         assertMemoryLeak(() -> {
@@ -549,6 +580,39 @@ public void testDoubleArrayWrapperMultipleRows() throws Exception {
         });
     }
 
+    @Test
+    public void testLongArrayWrapperMultipleRows() throws Exception {
+        assertMemoryLeak(() -> {
+            try (QwpTableBuffer table = new QwpTableBuffer("test");
+                 LongArray arr = new LongArray(3)) {
+                QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+
+                arr.append(10).append(20).append(30);
+                col.addLongArray(arr);
+                table.nextRow();
+
+                arr.append(40).append(50).append(60);
+                col.addLongArray(arr);
+                table.nextRow();
+
+                arr.append(70).append(80).append(90);
+                col.addLongArray(arr);
+                table.nextRow();
+
+                assertEquals(3, col.getValueCount());
+                long[] encoded = readLongArraysLikeEncoder(col);
+                assertArrayEquals(new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90}, encoded);
+
+                byte[] dims = col.getArrayDims();
+                int[] shapes = col.getArrayShapes();
+                for (int i = 0; i < 3; i++) {
+                    assertEquals(1, dims[i]);
+                    assertEquals(3, shapes[i]);
+                }
+            }
+        });
+    }
+
     @Test
     public void testDoubleArrayWrapperShrinkingSize() throws Exception {
         assertMemoryLeak(() -> {

From 9f888b9cb0360108ccf36a47fd1d1839990f1b19 Mon Sep 17 00:00:00 2001
From: Jaromir Hamala 
Date: Fri, 6 Mar 2026 17:42:00 +0100
Subject: [PATCH 167/230] reduce allocations when sending symbols

---
 .../cutlass/qwp/client/QwpUdpSender.java      | 928 +++++++++++-------
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |  56 +-
 .../cutlass/qwp/client/QwpUdpSenderTest.java  | 162 +++
 .../qwp/protocol/QwpTableBufferTest.java      |  99 +-
 4 files changed, 863 insertions(+), 382 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
index 1ec6c8b..eae1a8a 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
@@ -42,6 +42,7 @@
 import io.questdb.client.std.bytes.DirectByteSlice;
 import io.questdb.client.network.NetworkFacade;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.TestOnly;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -90,18 +91,18 @@ public class QwpUdpSender implements Sender {
     private final boolean trackDatagramEstimate;
     private final NativeSegmentList datagramSegments = new NativeSegmentList();
     private final CharSequenceObjHashMap tableBuffers;
-    private QwpTableBuffer.ColumnBuffer[] stagedColumns = new QwpTableBuffer.ColumnBuffer[8];
+    private InProgressColumnState[] inProgressColumns = new InProgressColumnState[8];
 
     private QwpTableBuffer.ColumnBuffer cachedTimestampColumn;
     private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn;
     private boolean closed;
     private long committedDatagramEstimate;
-    private int stagedRowValueCount;
+    private int inProgressRowValueCount;
     private QwpTableBuffer currentTableBuffer;
     private String currentTableName;
     private QwpTableBuffer.ColumnBuffer[] rowFillColumns = new QwpTableBuffer.ColumnBuffer[8];
     private int rowFillColumnCount;
-    private int stagedColumnCount;
+    private int inProgressColumnCount;
 
     public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) {
         this(nf, interfaceIPv4, sendToAddress, port, ttl, 0);
@@ -423,9 +424,7 @@ public void reset() {
         }
         currentTableBuffer = null;
         currentTableName = null;
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
-        clearStagedRow();
+        clearTransientRowState();
         resetCommittedDatagramEstimate();
     }
 
@@ -464,11 +463,10 @@ public Sender table(CharSequence tableName) {
         ensureNoInProgressRow("switch tables");
         if (trackDatagramEstimate && currentTableBuffer != null && currentTableBuffer.getRowCount() > 0) {
             flushSingleTable(currentTableName, currentTableBuffer);
+        } else {
+            clearTransientRowState();
+            resetCommittedDatagramEstimate();
         }
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
-        clearStagedRow();
-        resetCommittedDatagramEstimate();
 
         currentTableName = tableName.toString();
         currentTableBuffer = tableBuffers.get(currentTableName);
@@ -485,10 +483,10 @@ public Sender timestampColumn(CharSequence columnName, long value, ChronoUnit un
         checkTableSelected();
         try {
             if (unit == ChronoUnit.NANOS) {
-                stageTimestampColumnValue(columnName, TYPE_TIMESTAMP_NANOS, value, ENTRY_TIMESTAMP_COL_NANOS);
+                stageTimestampColumnValue(columnName, TYPE_TIMESTAMP_NANOS, value);
             } else {
                 long micros = toMicros(value, unit);
-                stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS);
+                stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros);
             }
         } catch (RuntimeException | Error e) {
             rollbackCurrentRowToCommittedState();
@@ -503,7 +501,7 @@ public Sender timestampColumn(CharSequence columnName, Instant value) {
         checkTableSelected();
         try {
             long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L;
-            stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros, ENTRY_TIMESTAMP_COL_MICROS);
+            stageTimestampColumnValue(columnName, TYPE_TIMESTAMP, micros);
         } catch (RuntimeException | Error e) {
             rollbackCurrentRowToCommittedState();
             throw e;
@@ -514,41 +512,89 @@ public Sender timestampColumn(CharSequence columnName, Instant value) {
     private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean nullable) {
         QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn(name, type);
         if (col == null && currentTableBuffer.getRowCount() > 0) {
-            flushCommittedRowsOfCurrentTable();
+            // schema change while having some rows accumulated -> we flush committed rows of the current table
+            // and start from scratch with the new schema
+
+            if (hasInProgressRow()) {
+                // the slowest case: we are adding a new column while the in-progress row already has some columns.
+                // we need to store the in-progress row in a temporary buffer, flush the committed rows, and then replay the staged row into the new schema
+                // assumption: this case is rare
+
+                snapshotCurrentRowIntoReplayBuffer();
+                rollbackCurrentRowToCommittedState(false);
+                flushCommittedRowsOfCurrentTable();
+                replaySnapshotAsInProgressRow();
+                stagedRow.clear();
+            } else {
+                flushCommittedRowsOfCurrentTable();
+            }
             col = currentTableBuffer.getExistingColumn(name, type);
         }
+
         if (col == null) {
             col = currentTableBuffer.getOrCreateColumn(name, type, nullable);
         }
         return col;
     }
 
+    private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence columnName) {
+        InProgressColumnState existing = findInProgressColumnState(column);
+        if (existing != null) {
+            if (columnName != null && columnName.isEmpty()) {
+                throw new LineSenderException("designated timestamp already set for current row");
+            }
+            throw new LineSenderException("column '" + columnName + "' already set for current row");
+        }
+        appendInProgressColumnState(column);
+    }
+
+    private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) {
+        ensureInProgressColumnCapacity(inProgressColumnCount + 1);
+        InProgressColumnState state = inProgressColumns[inProgressColumnCount];
+        if (state == null) {
+            state = new InProgressColumnState();
+            inProgressColumns[inProgressColumnCount] = state;
+        }
+        state.of(column);
+        inProgressColumnCount++;
+    }
+
+    private InProgressColumnState findInProgressColumnState(QwpTableBuffer.ColumnBuffer column) {
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            InProgressColumnState state = inProgressColumns[i];
+            if (state.column == column) {
+                return state;
+            }
+        }
+        return null;
+    }
+
     private void stageBooleanColumnValue(CharSequence name, boolean value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageBoolean(col, value);
+        beginColumnWrite(col, name);
+        col.addBoolean(value);
+        inProgressRowValueCount++;
     }
 
     private void stageDecimal128ColumnValue(CharSequence name, Decimal128 value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageDecimal128(col, value);
+        beginColumnWrite(col, name);
+        col.addDecimal128(value);
+        inProgressRowValueCount++;
     }
 
     private void stageDecimal256ColumnValue(CharSequence name, Decimal256 value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageDecimal256(col, value);
+        beginColumnWrite(col, name);
+        col.addDecimal256(value);
+        inProgressRowValueCount++;
     }
 
     private void stageDecimal64ColumnValue(CharSequence name, Decimal64 value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageDecimal64(col, value);
+        beginColumnWrite(col, name);
+        col.addDecimal64(value);
+        inProgressRowValueCount++;
     }
 
     private void stageDesignatedTimestampValue(long value, boolean nanos) {
@@ -564,61 +610,110 @@ private void stageDesignatedTimestampValue(long value, boolean nanos) {
             }
             col = cachedTimestampColumn;
         }
-        addStagedColumn(col);
-        stagedRow.stageLong(col, nanos ? ENTRY_AT_NANOS : ENTRY_AT_MICROS, value);
+        beginColumnWrite(col, "");
+        col.addLong(value);
     }
 
     private void stageDoubleArrayColumnValue(CharSequence name, Object value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true);
-        stagedRow.stageDoubleArray(col, value);
-        stagedRowValueCount++;
-        addStagedColumn(col);
+        beginColumnWrite(col, name);
+        appendDoubleArrayValue(col, value);
+        inProgressRowValueCount++;
     }
 
     private void stageDoubleColumnValue(CharSequence name, double value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageDouble(col, value);
+        beginColumnWrite(col, name);
+        col.addDouble(value);
+        inProgressRowValueCount++;
     }
 
     private void stageLongArrayColumnValue(CharSequence name, Object value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true);
-        stagedRow.stageLongArray(col, value);
-        stagedRowValueCount++;
-        addStagedColumn(col);
+        beginColumnWrite(col, name);
+        appendLongArrayValue(col, value);
+        inProgressRowValueCount++;
     }
 
     private void stageLongColumnValue(CharSequence name, long value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageLong(col, ENTRY_LONG, value);
+        beginColumnWrite(col, name);
+        col.addLong(value);
+        inProgressRowValueCount++;
     }
 
     private void stageStringColumnValue(CharSequence name, CharSequence value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageUtf8(col, ENTRY_STRING, value, 0);
+        beginColumnWrite(col, name);
+        col.addString(value);
+        inProgressRowValueCount++;
     }
 
     private void stageSymbolColumnValue(CharSequence name, CharSequence value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageUtf8(col, ENTRY_SYMBOL, value, estimateSymbolPayloadDelta(col, col.getValueCount(), value));
+        beginColumnWrite(col, name);
+        col.addSymbol(value);
+        inProgressRowValueCount++;
     }
 
-    private void stageTimestampColumnValue(CharSequence name, byte type, long value, byte entryKind) {
+    private void stageTimestampColumnValue(CharSequence name, byte type, long value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true);
-        stagedRowValueCount++;
-        addStagedColumn(col);
-        stagedRow.stageLong(col, entryKind, value);
+        beginColumnWrite(col, name);
+        col.addLong(value);
+        inProgressRowValueCount++;
+    }
+
+    private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) {
+        if (value instanceof double[] values) {
+            column.addDoubleArray(values);
+            return;
+        }
+        if (value instanceof double[][] values) {
+            column.addDoubleArray(values);
+            return;
+        }
+        if (value instanceof double[][][] values) {
+            column.addDoubleArray(values);
+            return;
+        }
+        if (value instanceof DoubleArray values) {
+            column.addDoubleArray(values);
+            return;
+        }
+        throw new LineSenderException("unsupported double array type");
+    }
+
+    private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) {
+        if (value instanceof long[] values) {
+            column.addLongArray(values);
+            return;
+        }
+        if (value instanceof long[][] values) {
+            column.addLongArray(values);
+            return;
+        }
+        if (value instanceof long[][][] values) {
+            column.addLongArray(values);
+            return;
+        }
+        if (value instanceof LongArray values) {
+            column.addLongArray(values);
+            return;
+        }
+        throw new LineSenderException("unsupported long array type");
+    }
+
+    private void stageNullArrayColumnValue(CharSequence name, byte type) {
+        checkNotClosed();
+        checkTableSelected();
+        QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true);
+        beginColumnWrite(col, name);
+        col.addNull();
+        inProgressRowValueCount++;
     }
 
     private void atMicros(long timestampMicros) {
-        if (stagedRowValueCount == 0) {
+        if (inProgressRowValueCount == 0) {
             throw new LineSenderException("no columns were provided");
         }
         try {
@@ -631,7 +726,7 @@ private void atMicros(long timestampMicros) {
     }
 
     private void atNanos(long timestampNanos) {
-        if (stagedRowValueCount == 0) {
+        if (inProgressRowValueCount == 0) {
             throw new LineSenderException("no columns were provided");
         }
         try {
@@ -655,21 +750,32 @@ private void checkTableSelected() {
         }
     }
 
-    private void clearStagedRow() {
-        stagedRow.clear();
-        stagedRowValueCount = 0;
-        stagedColumnCount = 0;
+    private void clearInProgressRow() {
+        inProgressRowValueCount = 0;
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            InProgressColumnState state = inProgressColumns[i];
+            if (state != null) {
+                state.clear();
+            }
+        }
+        inProgressColumnCount = 0;
         rowFillColumnCount = 0;
     }
 
     private void rollbackCurrentRowToCommittedState() {
+        rollbackCurrentRowToCommittedState(true);
+    }
+
+    private void rollbackCurrentRowToCommittedState(boolean clearReplayBuffer) {
         if (currentTableBuffer != null) {
             currentTableBuffer.cancelCurrentRow();
             currentTableBuffer.rollbackUncommittedColumns();
         }
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
-        clearStagedRow();
+        clearCachedTimestampColumns();
+        clearInProgressRow();
+        if (clearReplayBuffer) {
+            clearReplayBuffer();
+        }
     }
 
     private void collectRowFillColumns(int targetRows) {
@@ -688,7 +794,7 @@ private void collectRowFillColumns(int targetRows) {
     }
 
     private void commitCurrentRow() {
-        if (stagedRowValueCount == 0) {
+        if (inProgressRowValueCount == 0) {
             throw new LineSenderException("no columns were provided");
         }
 
@@ -696,26 +802,30 @@ private void commitCurrentRow() {
         int targetRows = currentTableBuffer.getRowCount() + 1;
         collectRowFillColumns(targetRows);
         if (trackDatagramEstimate) {
-            estimate = estimateCurrentDatagramSizeWithStagedRow(targetRows);
+            estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows);
             if (estimate > maxDatagramSize) {
                 if (currentTableBuffer.getRowCount() == 0) {
                     throw singleRowTooLarge(estimate);
                 }
+                snapshotCurrentRowIntoReplayBuffer();
+                rollbackCurrentRowToCommittedState(false);
                 flushCommittedRowsOfCurrentTable();
+                replaySnapshotAsInProgressRow();
+                stagedRow.clear();
                 targetRows = currentTableBuffer.getRowCount() + 1;
                 collectRowFillColumns(targetRows);
-                estimate = estimateCurrentDatagramSizeWithStagedRow(targetRows);
+                estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows);
                 if (estimate > maxDatagramSize) {
                     throw singleRowTooLarge(estimate);
                 }
             }
         }
 
-        commitStagedRow();
+        currentTableBuffer.nextRow(rowFillColumns, rowFillColumnCount);
         if (trackDatagramEstimate) {
             committedDatagramEstimate = estimate;
         }
-        clearStagedRow();
+        clearInProgressRow();
     }
 
     private void ensureNoInProgressRow(String operation) {
@@ -762,13 +872,14 @@ private long estimateBaseForCurrentSchema() {
         return estimate;
     }
 
-    private long estimateCurrentDatagramSizeWithStagedRow(int targetRows) {
+    private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) {
         long estimate = currentTableBuffer.getRowCount() > 0 ? committedDatagramEstimate : estimateBaseForCurrentSchema();
-        for (int i = 0, n = stagedRow.size(); i < n; i++) {
-            QwpTableBuffer.ColumnBuffer col = stagedRow.getColumn(i);
-            estimate += estimateStagedEntryPayload(i, col);
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            InProgressColumnState state = inProgressColumns[i];
+            QwpTableBuffer.ColumnBuffer col = state.column;
+            estimate += estimateInProgressColumnPayload(state);
             if (col.isNullable()) {
-                estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize());
+                estimate += bitmapBytes(targetRows) - bitmapBytes(state.sizeBefore);
             }
         }
         for (int i = 0; i < rowFillColumnCount; i++) {
@@ -783,19 +894,24 @@ private long estimateCurrentDatagramSizeWithStagedRow(int targetRows) {
         return estimate;
     }
 
-    private long estimateStagedEntryPayload(int entryIndex, QwpTableBuffer.ColumnBuffer col) {
-        int valueCountBefore = col.getValueCount();
-        return switch (stagedRow.getKind(entryIndex)) {
-            case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_DOUBLE, ENTRY_LONG,
-                    ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> 8;
-            case ENTRY_BOOL -> packedBytes(valueCountBefore + 1) - packedBytes(valueCountBefore);
-            case ENTRY_DECIMAL64 -> 8;
-            case ENTRY_DECIMAL128 -> 16;
-            case ENTRY_DECIMAL256 -> 32;
-            case ENTRY_DOUBLE_ARRAY, ENTRY_LONG_ARRAY -> stagedRow.getLong0(entryIndex);
-            case ENTRY_STRING -> stagedRow.getAuxInt(entryIndex) < 0 ? 0 : 4L + stagedRow.getAuxInt(entryIndex);
-            case ENTRY_SYMBOL -> stagedRow.getLong1(entryIndex);
-            default -> throw new LineSenderException("unknown staged row entry type: " + stagedRow.getKind(entryIndex));
+    private long estimateInProgressColumnPayload(InProgressColumnState state) {
+        QwpTableBuffer.ColumnBuffer col = state.column;
+        int valueCountBefore = state.valueCountBefore;
+        int valueCountAfter = col.getValueCount();
+        if (valueCountAfter == valueCountBefore) {
+            return 0;
+        }
+
+        return switch (col.getType()) {
+            case TYPE_BOOLEAN -> packedBytes(valueCountAfter) - packedBytes(valueCountBefore);
+            case TYPE_DECIMAL64 -> 8;
+            case TYPE_DECIMAL128 -> 16;
+            case TYPE_DECIMAL256 -> 32;
+            case TYPE_DOUBLE, TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> 8;
+            case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> estimateArrayPayloadBytes(col, state);
+            case TYPE_STRING, TYPE_VARCHAR -> 4L + (col.getStringDataSize() - state.stringDataSizeBefore);
+            case TYPE_SYMBOL -> estimateSymbolPayloadDelta(col, state);
+            default -> throw new LineSenderException("unsupported in-progress column type: " + col.getType());
         };
     }
 
@@ -814,34 +930,46 @@ private void ensureRowFillColumnCapacity(int required) {
         rowFillColumns = newArr;
     }
 
-    private void ensureStagedColumnCapacity(int required) {
-        if (required <= stagedColumns.length) {
+    private void ensureInProgressColumnCapacity(int required) {
+        if (required <= inProgressColumns.length) {
             return;
         }
 
-        int newCapacity = stagedColumns.length;
+        int newCapacity = inProgressColumns.length;
         while (newCapacity < required) {
             newCapacity *= 2;
         }
 
-        QwpTableBuffer.ColumnBuffer[] newArr = new QwpTableBuffer.ColumnBuffer[newCapacity];
-        System.arraycopy(stagedColumns, 0, newArr, 0, stagedColumnCount);
-        stagedColumns = newArr;
+        InProgressColumnState[] newArr = new InProgressColumnState[newCapacity];
+        System.arraycopy(inProgressColumns, 0, newArr, 0, inProgressColumnCount);
+        inProgressColumns = newArr;
+    }
+
+    private long estimateArrayPayloadBytes(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) {
+        int shapeCount = col.getArrayShapeOffset() - state.arrayShapeOffsetBefore;
+        int dataCount = col.getArrayDataOffset() - state.arrayDataOffsetBefore;
+        int elementSize = col.getType() == TYPE_LONG_ARRAY ? Long.BYTES : Double.BYTES;
+        return 1L + (long) shapeCount * Integer.BYTES + (long) dataCount * elementSize;
     }
 
-    private long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, int valueCountBefore, CharSequence value) {
-        if (value == null) {
+    private long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) {
+        int valueCountBefore = state.valueCountBefore;
+        int valueCountAfter = col.getValueCount();
+        if (valueCountAfter == valueCountBefore) {
             return 0;
         }
 
-        int dictSizeBefore = col.getSymbolDictionarySize();
-        if (col.hasSymbol(value)) {
-            int maxIndex = Math.max(0, dictSizeBefore - 1);
-            return NativeBufferWriter.varintSize(maxIndex);
+        int dictSizeBefore = state.symbolDictionarySizeBefore;
+        long dataAddress = col.getDataAddress();
+        int idx = Unsafe.getUnsafe().getInt(dataAddress + (long) valueCountBefore * Integer.BYTES);
+        int dictSizeAfter = col.getSymbolDictionarySize();
+
+        if (dictSizeAfter == dictSizeBefore) {
+            return NativeBufferWriter.varintSize(idx);
         }
 
-        int dictSizeAfter = dictSizeBefore + 1;
         long delta = 0;
+        CharSequence value = col.getSymbolValue(idx);
         int utf8Len = utf8Length(value);
         delta += NativeBufferWriter.varintSize(utf8Len) + utf8Len;
         delta += NativeBufferWriter.varintSize(dictSizeAfter) - NativeBufferWriter.varintSize(dictSizeBefore);
@@ -863,8 +991,7 @@ private void flushCommittedRowsOfCurrentTable() {
             return;
         }
         sendTableBuffer(currentTableName, currentTableBuffer);
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
+        clearCachedTimestampColumns();
         resetCommittedDatagramEstimate();
     }
 
@@ -881,43 +1008,68 @@ private void flushInternal() {
             }
             sendTableBuffer(tableName, tableBuffer);
         }
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
-        clearStagedRow();
+        clearTransientRowState();
         resetCommittedDatagramEstimate();
     }
 
     private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) {
         sendTableBuffer(tableName, tableBuffer);
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
-        clearStagedRow();
+        clearTransientRowState();
         resetCommittedDatagramEstimate();
     }
 
     private boolean hasInProgressRow() {
-        return stagedRow.size() > 0;
+        return inProgressColumnCount > 0;
     }
 
     private boolean isStagedColumn(QwpTableBuffer.ColumnBuffer col) {
-        for (int i = 0; i < stagedColumnCount; i++) {
-            if (stagedColumns[i] == col) {
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            InProgressColumnState state = inProgressColumns[i];
+            if (state != null && state.column == col) {
                 return true;
             }
         }
         return false;
     }
 
-    private void addStagedColumn(QwpTableBuffer.ColumnBuffer col) {
-        if (isStagedColumn(col)) {
-            return;
+    private void snapshotCurrentRowIntoReplayBuffer() {
+        stagedRow.clear();
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            InProgressColumnState state = inProgressColumns[i];
+            int replayKind;
+            if (state.column == cachedTimestampColumn) {
+                replayKind = ENTRY_AT_MICROS;
+            } else if (state.column == cachedTimestampNanosColumn) {
+                replayKind = ENTRY_AT_NANOS;
+            } else {
+                replayKind = 0;
+            }
+            stagedRow.snapshotInProgressColumn(state, replayKind);
         }
-        ensureStagedColumnCapacity(stagedColumnCount + 1);
-        stagedColumns[stagedColumnCount++] = col;
     }
 
-    private void commitStagedRow() {
-        stagedRow.commitIntoTableBuffer(currentTableBuffer, rowFillColumns, rowFillColumnCount);
+    private void replaySnapshotAsInProgressRow() {
+        clearInProgressRow();
+        cachedTimestampColumn = null;
+        cachedTimestampNanosColumn = null;
+        for (int i = 0, n = stagedRow.size(); i < n; i++) {
+            QwpTableBuffer.ColumnBuffer column = currentTableBuffer.getOrCreateColumn(
+                    stagedRow.getColumnName(i),
+                    stagedRow.getColumnType(i),
+                    stagedRow.isColumnNullable(i)
+            );
+            appendInProgressColumnState(column);
+            stagedRow.appendEntryIntoColumn(i, column);
+
+            int kind = stagedRow.getKind(i);
+            if (kind == ENTRY_AT_MICROS) {
+                cachedTimestampColumn = column;
+            } else if (kind == ENTRY_AT_NANOS) {
+                cachedTimestampNanosColumn = column;
+            } else {
+                inProgressRowValueCount++;
+            }
+        }
     }
 
     private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) {
@@ -947,6 +1099,37 @@ private void resetCommittedDatagramEstimate() {
         committedDatagramEstimate = 0;
     }
 
+    private void clearCachedTimestampColumns() {
+        cachedTimestampColumn = null;
+        cachedTimestampNanosColumn = null;
+    }
+
+    private void clearReplayBuffer() {
+        stagedRow.clear();
+    }
+
+    private void clearTransientRowState() {
+        clearCachedTimestampColumns();
+        clearInProgressRow();
+        clearReplayBuffer();
+    }
+
+    // Public test hooks because module boundaries prevent tests from sharing this package.
+    @TestOnly
+    public void stageNullDoubleArrayForTest(CharSequence name) {
+        stageNullArrayColumnValue(name, TYPE_DOUBLE_ARRAY);
+    }
+
+    @TestOnly
+    public void stageNullLongArrayForTest(CharSequence name) {
+        stageNullArrayColumnValue(name, TYPE_LONG_ARRAY);
+    }
+
+    @TestOnly
+    public QwpTableBuffer currentTableBufferForTest() {
+        return currentTableBuffer;
+    }
+
     private void sendTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer) {
         int payloadLength = encodeTablePayloadForUdp(tableBuffer);
         headerBuffer.reset();
@@ -1022,6 +1205,30 @@ private static int utf8Length(CharSequence s) {
         return len;
     }
 
+    private static final class InProgressColumnState {
+        private int arrayDataOffsetBefore;
+        private int arrayShapeOffsetBefore;
+        private QwpTableBuffer.ColumnBuffer column;
+        private int sizeBefore;
+        private long stringDataSizeBefore;
+        private int symbolDictionarySizeBefore;
+        private int valueCountBefore;
+
+        void clear() {
+            column = null;
+        }
+
+        void of(QwpTableBuffer.ColumnBuffer column) {
+            this.column = column;
+            this.sizeBefore = column.getSize();
+            this.valueCountBefore = column.getValueCount();
+            this.stringDataSizeBefore = column.getStringDataSize();
+            this.arrayShapeOffsetBefore = column.getArrayShapeOffset();
+            this.arrayDataOffsetBefore = column.getArrayDataOffset();
+            this.symbolDictionarySizeBefore = column.getSymbolDictionarySize();
+        }
+    }
+
     private static final class NativeRowStaging {
         private static final int AUX_INT_OFFSET = 4;
         private static final int ENTRY_SIZE = 40;
@@ -1035,79 +1242,59 @@ private static final class NativeRowStaging {
         private final Decimal64 decimal64Sink = new Decimal64();
         private final OffHeapAppendMemory entries = new OffHeapAppendMemory(ENTRY_SIZE * 8L);
         private final OffHeapAppendMemory varData = new OffHeapAppendMemory(128);
-        private QwpTableBuffer.ColumnBuffer[] columns = new QwpTableBuffer.ColumnBuffer[8];
+        private String[] columnNames = new String[8];
+        private byte[] columnTypes = new byte[8];
+        private boolean[] columnNullables = new boolean[8];
         private int size;
 
-        void stageBoolean(QwpTableBuffer.ColumnBuffer column, boolean value) {
-            stageEntry(column, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0);
-        }
-
-        void stageDecimal128(QwpTableBuffer.ColumnBuffer column, Decimal128 value) {
-            stageEntry(column, ENTRY_DECIMAL128, value.getScale(), value.getHigh(), value.getLow(), 0, 0);
-        }
-
-        void stageDecimal256(QwpTableBuffer.ColumnBuffer column, Decimal256 value) {
-            stageEntry(
-                    column,
-                    ENTRY_DECIMAL256,
-                    value.getScale(),
-                    value.getHh(),
-                    value.getHl(),
-                    value.getLh(),
-                    value.getLl()
-            );
-        }
-
-        void stageDecimal64(QwpTableBuffer.ColumnBuffer column, Decimal64 value) {
-            stageEntry(column, ENTRY_DECIMAL64, value.getScale(), value.getValue(), 0, 0, 0);
-        }
-
-        void stageDouble(QwpTableBuffer.ColumnBuffer column, double value) {
-            stageEntry(column, ENTRY_DOUBLE, 0, Double.doubleToRawLongBits(value), 0, 0, 0);
-        }
-
-        void stageDoubleArray(QwpTableBuffer.ColumnBuffer column, Object value) {
-            long offset = varData.getAppendOffset();
-            try {
-                appendDoubleArrayPayload(value);
-                long payloadLength = varData.getAppendOffset() - offset;
-                stageEntry(column, ENTRY_DOUBLE_ARRAY, 0, payloadLength, offset, 0, 0);
-            } catch (Throwable t) {
-                varData.jumpTo(offset);
-                throw t;
-            }
-        }
-
-        void stageLong(QwpTableBuffer.ColumnBuffer column, int kind, long value) {
-            stageEntry(column, kind, 0, value, 0, 0, 0);
-        }
-
-        void stageLongArray(QwpTableBuffer.ColumnBuffer column, Object value) {
-            long offset = varData.getAppendOffset();
-            try {
-                appendLongArrayPayload(value);
-                long payloadLength = varData.getAppendOffset() - offset;
-                stageEntry(column, ENTRY_LONG_ARRAY, 0, payloadLength, offset, 0, 0);
-            } catch (Throwable t) {
-                varData.jumpTo(offset);
-                throw t;
-            }
-        }
-
-        void stageUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value, long long1) {
-            int len = -1;
-            long offset = 0;
-            if (value != null) {
-                offset = varData.getAppendOffset();
-                varData.putUtf8(value);
-                len = (int) (varData.getAppendOffset() - offset);
+        void appendEntryIntoColumn(int index, QwpTableBuffer.ColumnBuffer column) {
+            long entryAddress = entryAddress(index);
+            int kind = Unsafe.getUnsafe().getInt(entryAddress);
+            int auxInt = Unsafe.getUnsafe().getInt(entryAddress + AUX_INT_OFFSET);
+            long long0 = Unsafe.getUnsafe().getLong(entryAddress + LONG0_OFFSET);
+            long long1 = Unsafe.getUnsafe().getLong(entryAddress + LONG1_OFFSET);
+            switch (kind) {
+                case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS ->
+                        column.addLong(long0);
+                case ENTRY_BOOL -> column.addBoolean(long0 != 0);
+                case ENTRY_DECIMAL64 -> {
+                    decimal64Sink.ofRaw(long0);
+                    decimal64Sink.setScale(auxInt);
+                    column.addDecimal64(decimal64Sink);
+                }
+                case ENTRY_DECIMAL128 -> {
+                    decimal128Sink.ofRaw(long0, long1);
+                    decimal128Sink.setScale(auxInt);
+                    column.addDecimal128(decimal128Sink);
+                }
+                case ENTRY_DECIMAL256 -> {
+                    decimal256Sink.ofRaw(
+                            long0,
+                            long1,
+                            Unsafe.getUnsafe().getLong(entryAddress + LONG2_OFFSET),
+                            Unsafe.getUnsafe().getLong(entryAddress + LONG3_OFFSET)
+                    );
+                    decimal256Sink.setScale(auxInt);
+                    column.addDecimal256(decimal256Sink);
+                }
+                case ENTRY_DOUBLE -> column.addDouble(Double.longBitsToDouble(long0));
+                case ENTRY_DOUBLE_ARRAY -> column.addDoubleArrayPayload(varData.addressOf(long1), long0);
+                case ENTRY_LONG_ARRAY -> column.addLongArrayPayload(varData.addressOf(long1), long0);
+                case ENTRY_STRING -> column.addStringUtf8(varData.addressOf(long0), auxInt);
+                case ENTRY_SYMBOL -> {
+                    if (auxInt < 0) {
+                        column.addSymbol(null);
+                    } else {
+                        column.addSymbolUtf8(varData.addressOf(long0), auxInt);
+                    }
+                }
+                default -> throw new LineSenderException("unknown staged row entry type: " + kind);
             }
-            stageEntry(column, kind, len, offset, long1, 0, 0);
         }
 
         void clear() {
             for (int i = 0; i < size; i++) {
-                columns[i] = null;
+                columnNames[i] = null;
             }
             size = 0;
             entries.truncate();
@@ -1119,97 +1306,97 @@ void close() {
             varData.close();
         }
 
-        int getAuxInt(int index) {
-            return Unsafe.getUnsafe().getInt(entryAddress(index) + AUX_INT_OFFSET);
+        byte getColumnType(int index) {
+            return columnTypes[index];
         }
 
-        QwpTableBuffer.ColumnBuffer getColumn(int index) {
-            return columns[index];
+        String getColumnName(int index) {
+            return columnNames[index];
         }
 
         int getKind(int index) {
             return Unsafe.getUnsafe().getInt(entryAddress(index));
         }
 
-        long getLong0(int index) {
-            return Unsafe.getUnsafe().getLong(entryAddress(index) + LONG0_OFFSET);
+        boolean isColumnNullable(int index) {
+            return columnNullables[index];
         }
 
-        long getLong1(int index) {
-            return Unsafe.getUnsafe().getLong(entryAddress(index) + LONG1_OFFSET);
-        }
+        void snapshotInProgressColumn(InProgressColumnState state, int replayKind) {
+            QwpTableBuffer.ColumnBuffer column = state.column;
+            int kind = replayKind != 0 ? replayKind : defaultKind(column.getType());
+            int valueIndex = state.valueCountBefore;
+            long dataAddress = column.getDataAddress();
 
-        void commitIntoTableBuffer(QwpTableBuffer tableBuffer, QwpTableBuffer.ColumnBuffer[] rowFillColumns, int rowFillColumnCount) {
-            for (int i = 0; i < size; i++) {
-                QwpTableBuffer.ColumnBuffer column = columns[i];
-                long entryAddress = entryAddress(i);
-                int kind = Unsafe.getUnsafe().getInt(entryAddress);
-                int auxInt = Unsafe.getUnsafe().getInt(entryAddress + AUX_INT_OFFSET);
-                long long0 = Unsafe.getUnsafe().getLong(entryAddress + LONG0_OFFSET);
-                long long1 = Unsafe.getUnsafe().getLong(entryAddress + LONG1_OFFSET);
-                switch (kind) {
-                    case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS ->
-                            column.addLong(long0);
-                    case ENTRY_BOOL -> column.addBoolean(long0 != 0);
-                    case ENTRY_DECIMAL64 -> {
-                        decimal64Sink.ofRaw(long0);
-                        decimal64Sink.setScale(auxInt);
-                        column.addDecimal64(decimal64Sink);
-                    }
-                    case ENTRY_DECIMAL128 -> {
-                        decimal128Sink.ofRaw(long0, long1);
-                        decimal128Sink.setScale(auxInt);
-                        column.addDecimal128(decimal128Sink);
-                    }
-                    case ENTRY_DECIMAL256 -> {
-                        decimal256Sink.ofRaw(
-                                long0,
-                                long1,
-                                Unsafe.getUnsafe().getLong(entryAddress + LONG2_OFFSET),
-                                Unsafe.getUnsafe().getLong(entryAddress + LONG3_OFFSET)
-                        );
-                        decimal256Sink.setScale(auxInt);
-                        column.addDecimal256(decimal256Sink);
-                    }
-                    case ENTRY_DOUBLE -> column.addDouble(Double.longBitsToDouble(long0));
-                    case ENTRY_DOUBLE_ARRAY -> column.addDoubleArrayPayload(varData.addressOf(long1), long0);
-                    case ENTRY_LONG_ARRAY -> column.addLongArrayPayload(varData.addressOf(long1), long0);
-                    case ENTRY_STRING -> column.addStringUtf8(varData.addressOf(long0), auxInt);
-                    case ENTRY_SYMBOL -> {
-                        if (auxInt < 0) {
-                            column.addSymbol(null);
-                        } else {
-                            column.addSymbolUtf8(varData.addressOf(long0), auxInt);
-                        }
-                    }
-                    default -> throw new LineSenderException("unknown staged row entry type: " + kind);
+            switch (kind) {
+                case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> {
+                    long value = Unsafe.getUnsafe().getLong(dataAddress + (long) valueIndex * Long.BYTES);
+                    stageEntry(column, kind, 0, value, 0, 0, 0);
+                }
+                case ENTRY_BOOL -> {
+                    boolean value = Unsafe.getUnsafe().getByte(dataAddress + valueIndex) != 0;
+                    stageEntry(column, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0);
+                }
+                case ENTRY_DECIMAL64 -> {
+                    long value = Unsafe.getUnsafe().getLong(dataAddress + (long) valueIndex * Long.BYTES);
+                    stageEntry(column, ENTRY_DECIMAL64, column.getDecimalScale(), value, 0, 0, 0);
+                }
+                case ENTRY_DECIMAL128 -> {
+                    long offset = (long) valueIndex * 16;
+                    stageEntry(
+                            column,
+                            ENTRY_DECIMAL128,
+                            column.getDecimalScale(),
+                            Unsafe.getUnsafe().getLong(dataAddress + offset),
+                            Unsafe.getUnsafe().getLong(dataAddress + offset + Long.BYTES),
+                            0,
+                            0
+                    );
                 }
+                case ENTRY_DECIMAL256 -> {
+                    long offset = (long) valueIndex * 32;
+                    stageEntry(
+                            column,
+                            ENTRY_DECIMAL256,
+                            column.getDecimalScale(),
+                            Unsafe.getUnsafe().getLong(dataAddress + offset),
+                            Unsafe.getUnsafe().getLong(dataAddress + offset + 8),
+                            Unsafe.getUnsafe().getLong(dataAddress + offset + 16),
+                            Unsafe.getUnsafe().getLong(dataAddress + offset + 24)
+                    );
+                }
+                case ENTRY_DOUBLE -> {
+                    long value = Unsafe.getUnsafe().getLong(dataAddress + (long) valueIndex * Double.BYTES);
+                    stageEntry(column, ENTRY_DOUBLE, 0, value, 0, 0, 0);
+                }
+                case ENTRY_DOUBLE_ARRAY -> snapshotDoubleArray(column, state);
+                case ENTRY_LONG_ARRAY -> snapshotLongArray(column, state);
+                case ENTRY_STRING -> snapshotString(column, state);
+                case ENTRY_SYMBOL -> snapshotSymbol(column, state);
+                default -> throw new LineSenderException("unsupported replay row entry type: " + kind);
             }
-            tableBuffer.nextRow(rowFillColumns, rowFillColumnCount);
         }
 
         int size() {
             return size;
         }
 
-        private void stageEntry(
-                QwpTableBuffer.ColumnBuffer column,
-                int kind,
-                int auxInt,
-                long long0,
-                long long1,
-                long long2,
-                long long3
-        ) {
-            ensureCapacity(size + 1);
-            columns[size] = column;
-            entries.putInt(kind);
-            entries.putInt(auxInt);
-            entries.putLong(long0);
-            entries.putLong(long1);
-            entries.putLong(long2);
-            entries.putLong(long3);
-            size++;
+        private int defaultKind(byte columnType) {
+            return switch (columnType) {
+                case TYPE_BOOLEAN -> ENTRY_BOOL;
+                case TYPE_DECIMAL128 -> ENTRY_DECIMAL128;
+                case TYPE_DECIMAL256 -> ENTRY_DECIMAL256;
+                case TYPE_DECIMAL64 -> ENTRY_DECIMAL64;
+                case TYPE_DOUBLE -> ENTRY_DOUBLE;
+                case TYPE_DOUBLE_ARRAY -> ENTRY_DOUBLE_ARRAY;
+                case TYPE_LONG -> ENTRY_LONG;
+                case TYPE_LONG_ARRAY -> ENTRY_LONG_ARRAY;
+                case TYPE_STRING, TYPE_VARCHAR -> ENTRY_STRING;
+                case TYPE_SYMBOL -> ENTRY_SYMBOL;
+                case TYPE_TIMESTAMP -> ENTRY_TIMESTAMP_COL_MICROS;
+                case TYPE_TIMESTAMP_NANOS -> ENTRY_TIMESTAMP_COL_NANOS;
+                default -> throw new LineSenderException("unsupported replay row column type: " + columnType);
+            };
         }
 
         private long entryAddress(int index) {
@@ -1217,151 +1404,156 @@ private long entryAddress(int index) {
         }
 
         private void ensureCapacity(int required) {
-            if (required <= columns.length) {
+            if (required <= columnNames.length) {
                 return;
             }
 
-            int newCapacity = columns.length;
+            int newCapacity = columnNames.length;
             while (newCapacity < required) {
                 newCapacity *= 2;
             }
 
-            QwpTableBuffer.ColumnBuffer[] newColumns = new QwpTableBuffer.ColumnBuffer[newCapacity];
-            System.arraycopy(columns, 0, newColumns, 0, size);
-            columns = newColumns;
-        }
+            String[] newColumnNames = new String[newCapacity];
+            System.arraycopy(columnNames, 0, newColumnNames, 0, size);
+            columnNames = newColumnNames;
 
-        private static int checkedArrayElementCount(long product) {
-            if (product > Integer.MAX_VALUE) {
-                throw new LineSenderException("array too large: total element count exceeds int range");
-            }
-            return (int) product;
+            byte[] newColumnTypes = new byte[newCapacity];
+            System.arraycopy(columnTypes, 0, newColumnTypes, 0, size);
+            columnTypes = newColumnTypes;
+
+            boolean[] newColumnNullables = new boolean[newCapacity];
+            System.arraycopy(columnNullables, 0, newColumnNullables, 0, size);
+            columnNullables = newColumnNullables;
         }
 
-        private void appendDoubleArrayPayload(Object value) {
-            if (value instanceof double[] values) {
-                varData.putByte((byte) 1);
-                varData.putInt(values.length);
-                for (double v : values) {
-                    varData.putDouble(v);
-                }
+        private void snapshotDoubleArray(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
+            if (column.getValueCount() == state.valueCountBefore) {
+                stageEntry(column, ENTRY_DOUBLE_ARRAY, 0, -1, 0, 0, 0);
                 return;
             }
-            if (value instanceof double[][] values) {
-                int dim0 = values.length;
-                int dim1 = dim0 > 0 ? values[0].length : 0;
-                for (int i = 1; i < dim0; i++) {
-                    if (values[i].length != dim1) {
-                        throw new LineSenderException("irregular array shape");
-                    }
+
+            long offset = varData.getAppendOffset();
+            try {
+                int shapeStart = state.arrayShapeOffsetBefore;
+                int shapeEnd = column.getArrayShapeOffset();
+                int dataStart = state.arrayDataOffsetBefore;
+                int dataEnd = column.getArrayDataOffset();
+                int[] shapes = column.getArrayShapes();
+                double[] data = column.getDoubleArrayData();
+
+                varData.putByte((byte) (shapeEnd - shapeStart));
+                for (int i = shapeStart; i < shapeEnd; i++) {
+                    varData.putInt(shapes[i]);
                 }
-                checkedArrayElementCount((long) dim0 * dim1);
-                varData.putByte((byte) 2);
-                varData.putInt(dim0);
-                varData.putInt(dim1);
-                for (double[] row : values) {
-                    for (double v : row) {
-                        varData.putDouble(v);
-                    }
+                for (int i = dataStart; i < dataEnd; i++) {
+                    varData.putDouble(data[i]);
                 }
+
+                long payloadLength = varData.getAppendOffset() - offset;
+                stageEntry(column, ENTRY_DOUBLE_ARRAY, 0, payloadLength, offset, 0, 0);
+            } catch (Throwable t) {
+                varData.jumpTo(offset);
+                throw t;
+            }
+        }
+
+        private void snapshotLongArray(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
+            if (column.getValueCount() == state.valueCountBefore) {
+                stageEntry(column, ENTRY_LONG_ARRAY, 0, -1, 0, 0, 0);
                 return;
             }
-            if (value instanceof double[][][] values) {
-                int dim0 = values.length;
-                int dim1 = dim0 > 0 ? values[0].length : 0;
-                int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0;
-                for (int i = 0; i < dim0; i++) {
-                    if (values[i].length != dim1) {
-                        throw new LineSenderException("irregular array shape");
-                    }
-                    for (int j = 0; j < dim1; j++) {
-                        if (values[i][j].length != dim2) {
-                            throw new LineSenderException("irregular array shape");
-                        }
-                    }
+
+            long offset = varData.getAppendOffset();
+            try {
+                int shapeStart = state.arrayShapeOffsetBefore;
+                int shapeEnd = column.getArrayShapeOffset();
+                int dataStart = state.arrayDataOffsetBefore;
+                int dataEnd = column.getArrayDataOffset();
+                int[] shapes = column.getArrayShapes();
+                long[] data = column.getLongArrayData();
+
+                varData.putByte((byte) (shapeEnd - shapeStart));
+                for (int i = shapeStart; i < shapeEnd; i++) {
+                    varData.putInt(shapes[i]);
                 }
-                checkedArrayElementCount((long) dim0 * dim1 * dim2);
-                varData.putByte((byte) 3);
-                varData.putInt(dim0);
-                varData.putInt(dim1);
-                varData.putInt(dim2);
-                for (double[][] plane : values) {
-                    for (double[] row : plane) {
-                        for (double v : row) {
-                            varData.putDouble(v);
-                        }
-                    }
+                for (int i = dataStart; i < dataEnd; i++) {
+                    varData.putLong(data[i]);
                 }
-                return;
-            }
-            if (value instanceof DoubleArray values) {
-                values.appendToBufPtr(varData);
-                return;
+
+                long payloadLength = varData.getAppendOffset() - offset;
+                stageEntry(column, ENTRY_LONG_ARRAY, 0, payloadLength, offset, 0, 0);
+            } catch (Throwable t) {
+                varData.jumpTo(offset);
+                throw t;
             }
-            throw new LineSenderException("unsupported double array type");
         }
 
-        private void appendLongArrayPayload(Object value) {
-            if (value instanceof long[] values) {
-                varData.putByte((byte) 1);
-                varData.putInt(values.length);
-                for (long v : values) {
-                    varData.putLong(v);
-                }
+        private void snapshotString(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
+            if (column.getValueCount() == state.valueCountBefore) {
+                stageEntry(column, ENTRY_STRING, -1, 0, 0, 0, 0);
                 return;
             }
-            if (value instanceof long[][] values) {
-                int dim0 = values.length;
-                int dim1 = dim0 > 0 ? values[0].length : 0;
-                for (int i = 1; i < dim0; i++) {
-                    if (values[i].length != dim1) {
-                        throw new LineSenderException("irregular array shape");
-                    }
-                }
-                checkedArrayElementCount((long) dim0 * dim1);
-                varData.putByte((byte) 2);
-                varData.putInt(dim0);
-                varData.putInt(dim1);
-                for (long[] row : values) {
-                    for (long v : row) {
-                        varData.putLong(v);
-                    }
+
+            long offsetsAddress = column.getStringOffsetsAddress();
+            int start = Unsafe.getUnsafe().getInt(offsetsAddress + (long) state.valueCountBefore * Integer.BYTES);
+            int end = Unsafe.getUnsafe().getInt(offsetsAddress + (long) (state.valueCountBefore + 1) * Integer.BYTES);
+            int len = end - start;
+
+            long offset = varData.getAppendOffset();
+            try {
+                if (len > 0) {
+                    varData.putBlockOfBytes(column.getStringDataAddress() + start, len);
                 }
-                return;
+                stageEntry(column, ENTRY_STRING, len, offset, 0, 0, 0);
+            } catch (Throwable t) {
+                varData.jumpTo(offset);
+                throw t;
             }
-            if (value instanceof long[][][] values) {
-                int dim0 = values.length;
-                int dim1 = dim0 > 0 ? values[0].length : 0;
-                int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0;
-                for (int i = 0; i < dim0; i++) {
-                    if (values[i].length != dim1) {
-                        throw new LineSenderException("irregular array shape");
-                    }
-                    for (int j = 0; j < dim1; j++) {
-                        if (values[i][j].length != dim2) {
-                            throw new LineSenderException("irregular array shape");
-                        }
-                    }
-                }
-                checkedArrayElementCount((long) dim0 * dim1 * dim2);
-                varData.putByte((byte) 3);
-                varData.putInt(dim0);
-                varData.putInt(dim1);
-                varData.putInt(dim2);
-                for (long[][] plane : values) {
-                    for (long[] row : plane) {
-                        for (long v : row) {
-                            varData.putLong(v);
-                        }
-                    }
-                }
+        }
+
+        private void snapshotSymbol(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
+            if (column.getValueCount() == state.valueCountBefore) {
+                stageEntry(column, ENTRY_SYMBOL, -1, 0, 0, 0, 0);
                 return;
             }
-            if (value instanceof LongArray values) {
-                values.appendToBufPtr(varData);
-                return;
+
+            int valueIndex = state.valueCountBefore;
+            int localIndex = Unsafe.getUnsafe().getInt(column.getDataAddress() + (long) valueIndex * Integer.BYTES);
+            CharSequence value = column.getSymbolValue(localIndex);
+            stageUtf8(column, ENTRY_SYMBOL, value);
+        }
+
+        private void stageEntry(
+                QwpTableBuffer.ColumnBuffer column,
+                int kind,
+                int auxInt,
+                long long0,
+                long long1,
+                long long2,
+                long long3
+        ) {
+            ensureCapacity(size + 1);
+            columnNames[size] = column.getName();
+            columnTypes[size] = column.getType();
+            columnNullables[size] = column.isNullable();
+            entries.putInt(kind);
+            entries.putInt(auxInt);
+            entries.putLong(long0);
+            entries.putLong(long1);
+            entries.putLong(long2);
+            entries.putLong(long3);
+            size++;
+        }
+
+        private void stageUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value) {
+            int len = -1;
+            long offset = 0;
+            if (value != null) {
+                offset = varData.getAppendOffset();
+                varData.putUtf8(value);
+                len = (int) (varData.getAppendOffset() - offset);
             }
-            throw new LineSenderException("unsupported long array type");
+            stageEntry(column, kind, len, offset, 0, 0, 0);
         }
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index 55090eb..d6366cb 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -41,6 +41,7 @@
 import io.questdb.client.std.QuietCloseable;
 import io.questdb.client.std.Unsafe;
 import io.questdb.client.std.Vect;
+import io.questdb.client.std.str.StringSink;
 import io.questdb.client.std.str.Utf8s;
 
 import java.util.Arrays;
@@ -522,6 +523,7 @@ public static class ColumnBuffer implements QuietCloseable {
         private OffHeapAppendMemory stringOffsets;
         // Symbol specific (dictionary stays on-heap)
         private CharSequenceIntHashMap symbolDict;
+        private StringSink symbolLookupSink;
         private ObjList symbolList;
         private int valueCount;   // Actual stored values (excludes nulls)
 
@@ -1004,12 +1006,7 @@ public void addSymbol(CharSequence value) {
                 return;
             }
             ensureNullBitmapForNonNull();
-            int idx = symbolDict.get(value);
-            if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
-                idx = symbolList.size();
-                symbolDict.put(value, idx);
-                symbolList.add(Chars.toString(value));
-            }
+            int idx = getOrAddLocalSymbol(value);
             dataBuffer.putInt(idx);
             valueCount++;
             size++;
@@ -1020,7 +1017,22 @@ public void addSymbolUtf8(long ptr, int len) {
                 addNull();
                 return;
             }
-            addSymbol(Utf8s.stringFromUtf8Bytes(ptr, ptr + len));
+            ensureNullBitmapForNonNull();
+            StringSink lookupSink = symbolLookupSink;
+            if (lookupSink == null) {
+                symbolLookupSink = lookupSink = new StringSink(Math.max(16, len));
+            } else {
+                lookupSink.clear();
+            }
+            if (!Utf8s.utf8ToUtf16(ptr, ptr + len, lookupSink)) {
+                // Reuse the existing error path with the same diagnostic payload.
+                Utf8s.stringFromUtf8Bytes(ptr, ptr + len);
+                throw new AssertionError("unreachable");
+            }
+            int idx = getOrAddLocalSymbol(lookupSink);
+            dataBuffer.putInt(idx);
+            valueCount++;
+            size++;
         }
 
         public void addSymbolWithGlobalId(String value, int globalId) {
@@ -1029,12 +1041,7 @@ public void addSymbolWithGlobalId(String value, int globalId) {
                 return;
             }
             ensureNullBitmapForNonNull();
-            int localIdx = symbolDict.get(value);
-            if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
-                localIdx = symbolList.size();
-                symbolDict.put(value, localIdx);
-                symbolList.add(value);
-            }
+            int localIdx = getOrAddLocalSymbol(value);
             dataBuffer.putInt(localIdx);
 
             if (auxBuffer == null) {
@@ -1050,6 +1057,17 @@ public void addSymbolWithGlobalId(String value, int globalId) {
             size++;
         }
 
+        private int getOrAddLocalSymbol(CharSequence value) {
+            int idx = symbolDict.get(value);
+            if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
+                String symbol = Chars.toString(value);
+                idx = symbolList.size();
+                symbolDict.put(symbol, idx);
+                symbolList.add(symbol);
+            }
+            return idx;
+        }
+
         public void addUuid(long high, long low) {
             ensureNullBitmapForNonNull();
             // Store in wire order: lo first, hi second
@@ -1088,6 +1106,14 @@ public byte[] getArrayDims() {
             return arrayDims;
         }
 
+        public int getArrayDataOffset() {
+            return arrayDataOffset;
+        }
+
+        public int getArrayShapeOffset() {
+            return arrayShapeOffset;
+        }
+
         public int[] getArrayShapes() {
             return arrayShapes;
         }
@@ -1196,6 +1222,10 @@ public int getSymbolDictionarySize() {
             return symbolList != null ? symbolList.size() : 0;
         }
 
+        public CharSequence getSymbolValue(int index) {
+            return symbolList != null ? symbolList.getQuick(index) : null;
+        }
+
         public boolean hasSymbol(CharSequence value) {
             return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE;
         }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
index 0b9bc73..7bf98f9 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
@@ -29,6 +29,7 @@
 import io.questdb.client.cutlass.line.array.LongArray;
 import io.questdb.client.cutlass.qwp.client.QwpUdpSender;
 import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
+import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
 import io.questdb.client.network.NetworkFacadeImpl;
 import io.questdb.client.std.Decimal128;
 import io.questdb.client.std.Decimal256;
@@ -799,6 +800,127 @@ public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplayWithoutData
         });
     }
 
+    @Test
+    public void testSchemaChangeMidRowPreservesExistingStringAndSymbolValues() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .stringColumn("s", "alpha")
+                        .symbol("sym", "one")
+                        .atNow();
+
+                sender.longColumn("a", 2);
+                sender.stringColumn("s", "beta");
+                sender.symbol("sym", "two");
+                sender.longColumn("b", 3);
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L, "s", "alpha", "sym", "one"),
+                    decodedRow("t", "a", 2L, "s", "beta", "sym", "two", "b", 3L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testSchemaChangeMidRowPreservesExistingArrayValues() throws Exception {
+        assertMemoryLeak(() -> {
+            double[] first = {1.0, 2.0};
+            double[] second = {3.5, 4.5, 5.5};
+
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .doubleArray("da", first)
+                        .atNow();
+
+                sender.longColumn("a", 2);
+                sender.doubleArray("da", second);
+                sender.longColumn("b", 3);
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L, "da", doubleArrayValue(shape(2), first)),
+                    decodedRow("t", "a", 2L, "da", doubleArrayValue(shape(3), second), "b", 3L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testNullableArrayReplayKeepsNullArrayStateWithoutReflection() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longArray("la", new long[]{1, 2})
+                        .doubleArray("da", new double[]{1.0, 2.0})
+                        .atNow();
+
+                sender.stageNullLongArrayForTest("la");
+                sender.stageNullDoubleArrayForTest("da");
+                sender.longColumn("b", 3);
+
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.atNow();
+
+                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
+                Assert.assertNotNull(tableBuffer);
+                Assert.assertEquals(1, tableBuffer.getRowCount());
+
+                assertNullableArrayNullState(tableBuffer.getExistingColumn("la", TYPE_LONG_ARRAY));
+                assertNullableArrayNullState(tableBuffer.getExistingColumn("da", TYPE_DOUBLE_ARRAY));
+
+                QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("b", TYPE_LONG);
+                Assert.assertNotNull(longColumn);
+                Assert.assertEquals(1, longColumn.getSize());
+                Assert.assertEquals(1, longColumn.getValueCount());
+                Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress()));
+            }
+        });
+    }
+
+    @Test
+    public void testDuplicateColumnAfterSchemaFlushReplayIsRejected() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .atNow();
+
+                sender.longColumn("a", 2);
+                sender.longColumn("b", 3);
+                Assert.assertEquals(1, nf.sendCount);
+
+                assertThrowsContains("column 'a' already set for current row", () -> sender.longColumn("a", 4));
+
+                sender.cancelRow();
+                sender.longColumn("a", 5).atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L),
+                    decodedRow("t", "a", 5L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
     @Test
     public void testCancelRowAfterMidRowSchemaChangeDoesNotLeakSchema() throws Exception {
         assertMemoryLeak(() -> {
@@ -876,6 +998,36 @@ public void testUtf8StringAndSymbolStagingSupportsCancelAndPacketSizing() throws
         });
     }
 
+    @Test
+    public void testRepeatedUtf8SymbolsAcrossRowsPreserveRows() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("sym", sender -> sender.table("sym")
+                                    .longColumn("x", 1)
+                                    .symbol("sym", "東京")
+                                    .atNow(),
+                            "x", 1L,
+                            "sym", "東京"),
+                    row("sym", sender -> sender.table("sym")
+                                    .longColumn("x", 2)
+                                    .symbol("sym", "東京")
+                                    .atNow(),
+                            "x", 2L,
+                            "sym", "東京"),
+                    row("sym", sender -> sender.table("sym")
+                                    .longColumn("x", 3)
+                                    .symbol("sym", "Αθηνα")
+                                    .atNow(),
+                            "x", 3L,
+                            "sym", "Αθηνα")
+            );
+
+            RunResult result = runScenario(rows, 1024 * 1024);
+            Assert.assertEquals(1, result.sendCount);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
     @Test
     public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() throws Exception {
         assertMemoryLeak(() -> {
@@ -1236,6 +1388,16 @@ private static void assertThrowsContains(String expected, ThrowingRunnable runna
         }
     }
 
+    private static void assertNullableArrayNullState(QwpTableBuffer.ColumnBuffer column) {
+        Assert.assertNotNull(column);
+        Assert.assertEquals(1, column.getSize());
+        Assert.assertEquals(0, column.getValueCount());
+        Assert.assertTrue(column.isNullable());
+        Assert.assertTrue(column.isNull(0));
+        Assert.assertEquals(0, column.getArrayShapeOffset());
+        Assert.assertEquals(0, column.getArrayDataOffset());
+    }
+
     private static BigDecimal decimal(long unscaled, int scale) {
         return BigDecimal.valueOf(unscaled, scale);
     }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
index eead658..f7d9e74 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
@@ -24,17 +24,21 @@
 
 package io.questdb.client.test.cutlass.qwp.protocol;
 
+import io.questdb.client.cairo.CairoException;
 import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.line.array.DoubleArray;
 import io.questdb.client.cutlass.line.array.LongArray;
 import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
-import io.questdb.client.std.Unsafe;
 import io.questdb.client.std.Decimal128;
 import io.questdb.client.std.Decimal64;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
 import org.junit.Test;
 
+import java.nio.charset.StandardCharsets;
+
 import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
 import static org.junit.Assert.*;
 
@@ -403,6 +407,81 @@ public void testCancelRowResetsSymbolDictOnLateAddedColumn() throws Exception {
         });
     }
 
+    @Test
+    public void testAddSymbolUtf8ReusesExistingDictionaryEntry() throws Exception {
+        assertMemoryLeak(() -> {
+            try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+                QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true);
+                addSymbolUtf8(col, "東京");
+                table.nextRow();
+
+                addSymbolUtf8(col, "東京");
+                table.nextRow();
+
+                addSymbolUtf8(col, "Αθηνα");
+                table.nextRow();
+
+                assertEquals(3, col.getSize());
+                assertEquals(3, col.getValueCount());
+                assertEquals(2, col.getSymbolDictionarySize());
+                assertArrayEquals(new String[]{"東京", "Αθηνα"}, col.getSymbolDictionary());
+
+                long dataAddress = col.getDataAddress();
+                assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress));
+                assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress + 4));
+                assertEquals(1, Unsafe.getUnsafe().getInt(dataAddress + 8));
+            }
+        });
+    }
+
+    @Test
+    public void testAddSymbolUtf8CancelRowRewindsDictionary() throws Exception {
+        assertMemoryLeak(() -> {
+            try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+                table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0);
+                table.nextRow();
+
+                table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1);
+                QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true);
+                addSymbolUtf8(col, "stale");
+                table.cancelCurrentRow();
+
+                table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1);
+                addSymbolUtf8(col, "fresh");
+                table.nextRow();
+
+                assertEquals(2, col.getSize());
+                assertEquals(1, col.getValueCount());
+                assertArrayEquals(new String[]{"fresh"}, col.getSymbolDictionary());
+                assertEquals(0, Unsafe.getUnsafe().getInt(col.getDataAddress()));
+            }
+        });
+    }
+
+    @Test
+    public void testAddSymbolUtf8RejectsInvalidUtf8() throws Exception {
+        assertMemoryLeak(() -> {
+            try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+                QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true);
+                byte[] invalid = {(byte) 0xC3, 0x28};
+                long ptr = copyToNative(invalid);
+                try {
+                    try {
+                        col.addSymbolUtf8(ptr, invalid.length);
+                        fail("Expected CairoException");
+                    } catch (CairoException ex) {
+                        assertTrue(ex.getFlyweightMessage().toString().contains("cannot convert invalid UTF-8 sequence"));
+                    }
+                    assertEquals(0, col.getSize());
+                    assertEquals(0, col.getValueCount());
+                    assertEquals(0, col.getSymbolDictionarySize());
+                } finally {
+                    Unsafe.free(ptr, invalid.length, MemoryTag.NATIVE_DEFAULT);
+                }
+            }
+        });
+    }
+
     @Test
     public void testCancelRowRewindsDoubleArrayOffsets() throws Exception {
         assertMemoryLeak(() -> {
@@ -1050,4 +1129,22 @@ private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col)
         }
         return result;
     }
+
+    private static void addSymbolUtf8(QwpTableBuffer.ColumnBuffer col, String value) {
+        byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+        long ptr = copyToNative(bytes);
+        try {
+            col.addSymbolUtf8(ptr, bytes.length);
+        } finally {
+            Unsafe.free(ptr, bytes.length, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
+    private static long copyToNative(byte[] bytes) {
+        long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT);
+        for (int i = 0; i < bytes.length; i++) {
+            Unsafe.getUnsafe().putByte(ptr + i, bytes[i]);
+        }
+        return ptr;
+    }
 }

From 872e0c759054f4ff680c7677d42f2462b64fed04 Mon Sep 17 00:00:00 2001
From: Jaromir Hamala 
Date: Sat, 7 Mar 2026 12:39:12 +0100
Subject: [PATCH 168/230] optimizations

---
 .../cutlass/qwp/client/QwpColumnWriter.java   |  56 +-
 .../cutlass/qwp/client/QwpUdpSender.java      | 683 ++++++------------
 .../cutlass/qwp/protocol/QwpTableBuffer.java  | 191 +++++
 .../cutlass/qwp/client/QwpUdpSenderTest.java  | 276 +++++++
 .../qwp/protocol/QwpTableBufferTest.java      |  67 ++
 5 files changed, 805 insertions(+), 468 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java
index d1ab0a4..81c95e0 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java
@@ -46,8 +46,16 @@ class QwpColumnWriter {
     private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder();
     private QwpBufferWriter buffer;
 
-    private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla, boolean useGlobalSymbols) {
-        int valueCount = col.getValueCount();
+    private void encodeColumn(
+            QwpTableBuffer.ColumnBuffer col,
+            QwpColumnDef colDef,
+            int rowCount,
+            int valueCount,
+            long stringDataSize,
+            int symbolDictionarySize,
+            boolean useGorilla,
+            boolean useGlobalSymbols
+    ) {
         long dataAddr = col.getDataAddress();
 
         if (colDef.isNullable()) {
@@ -80,13 +88,13 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef,
                 break;
             case TYPE_STRING:
             case TYPE_VARCHAR:
-                writeStringColumn(col, valueCount);
+                writeStringColumn(col, valueCount, stringDataSize);
                 break;
             case TYPE_SYMBOL:
                 if (useGlobalSymbols) {
                     writeSymbolColumnWithGlobalIds(col, valueCount);
                 } else {
-                    writeSymbolColumn(col, valueCount);
+                    writeSymbolColumn(col, valueCount, symbolDictionarySize);
                 }
                 break;
             case TYPE_UUID:
@@ -118,8 +126,20 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef,
     }
 
     void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols, boolean useGorilla) {
+        encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useSchemaRef, useGlobalSymbols, useGorilla);
+    }
+
+    void encodeTable(
+            QwpTableBuffer tableBuffer,
+            int rowCount,
+            int[] limitedValueCounts,
+            long[] limitedStringDataSizes,
+            int[] limitedSymbolDictionarySizes,
+            boolean useSchemaRef,
+            boolean useGlobalSymbols,
+            boolean useGorilla
+    ) {
         QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs();
-        int rowCount = tableBuffer.getRowCount();
 
         if (useSchemaRef) {
             writeTableHeaderWithSchemaRef(
@@ -135,7 +155,17 @@ void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGl
         for (int i = 0; i < tableBuffer.getColumnCount(); i++) {
             QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i);
             QwpColumnDef colDef = columnDefs[i];
-            encodeColumn(col, colDef, rowCount, useGorilla, useGlobalSymbols);
+            int valueCount = col.getValueCount();
+            long stringDataSize = col.getStringDataSize();
+            int symbolDictionarySize = col.getSymbolDictionarySize();
+
+            if (limitedValueCounts != null && limitedValueCounts[i] > -1) {
+                valueCount = limitedValueCounts[i];
+                stringDataSize = limitedStringDataSizes[i];
+                symbolDictionarySize = limitedSymbolDictionarySizes[i];
+            }
+
+            encodeColumn(col, colDef, rowCount, valueCount, stringDataSize, symbolDictionarySize, useGorilla, useGlobalSymbols);
         }
     }
 
@@ -261,18 +291,16 @@ private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) {
         }
     }
 
-    private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) {
+    private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount, long stringDataSize) {
         buffer.putBlockOfBytes(col.getStringOffsetsAddress(), (long) (valueCount + 1) * 4);
-        buffer.putBlockOfBytes(col.getStringDataAddress(), col.getStringDataSize());
+        buffer.putBlockOfBytes(col.getStringDataAddress(), stringDataSize);
     }
 
-    private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) {
+    private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count, int dictionarySize) {
         long dataAddr = col.getDataAddress();
-        String[] dictionary = col.getSymbolDictionary();
-
-        buffer.putVarint(dictionary.length);
-        for (String symbol : dictionary) {
-            buffer.putString(symbol);
+        buffer.putVarint(dictionarySize);
+        for (int i = 0; i < dictionarySize; i++) {
+            buffer.putString((String) col.getSymbolValue(i));
         }
 
         for (int i = 0; i < count; i++) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
index eae1a8a..5cd63bb 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
@@ -29,7 +29,6 @@
 import io.questdb.client.cutlass.line.array.DoubleArray;
 import io.questdb.client.cutlass.line.array.LongArray;
 import io.questdb.client.cutlass.line.udp.UdpLineChannel;
-import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory;
 import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
 import io.questdb.client.std.CharSequenceObjHashMap;
@@ -48,6 +47,7 @@
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
 
 import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
 
@@ -64,20 +64,6 @@
  * the row back into column storage.
  */
 public class QwpUdpSender implements Sender {
-    private static final byte ENTRY_AT_MICROS = 1;
-    private static final byte ENTRY_AT_NANOS = 2;
-    private static final byte ENTRY_BOOL = 3;
-    private static final byte ENTRY_DECIMAL128 = 4;
-    private static final byte ENTRY_DECIMAL256 = 5;
-    private static final byte ENTRY_DECIMAL64 = 6;
-    private static final byte ENTRY_DOUBLE = 7;
-    private static final byte ENTRY_DOUBLE_ARRAY = 8;
-    private static final byte ENTRY_LONG = 9;
-    private static final byte ENTRY_LONG_ARRAY = 10;
-    private static final byte ENTRY_STRING = 11;
-    private static final byte ENTRY_SYMBOL = 12;
-    private static final byte ENTRY_TIMESTAMP_COL_MICROS = 13;
-    private static final byte ENTRY_TIMESTAMP_COL_NANOS = 14;
     private static final int VARINT_INT_UPPER_BOUND = 5;
     private static final int SAFETY_MARGIN_BYTES = 8;
     private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class);
@@ -87,7 +73,6 @@ public class QwpUdpSender implements Sender {
     private final NativeBufferWriter headerBuffer = new NativeBufferWriter();
     private final int maxDatagramSize;
     private final SegmentedNativeBufferWriter payloadWriter = new SegmentedNativeBufferWriter();
-    private final NativeRowStaging stagedRow = new NativeRowStaging();
     private final boolean trackDatagramEstimate;
     private final NativeSegmentList datagramSegments = new NativeSegmentList();
     private final CharSequenceObjHashMap tableBuffers;
@@ -100,7 +85,16 @@ public class QwpUdpSender implements Sender {
     private int inProgressRowValueCount;
     private QwpTableBuffer currentTableBuffer;
     private String currentTableName;
+    private int[] prefixArrayDataOffsetBefore = new int[8];
+    private int[] prefixArrayShapeOffsetBefore = new int[8];
+    private long[] prefixStringDataSizeBefore = new long[8];
+    private int[] prefixSymbolDictionarySizeBefore = new int[8];
+    private int[] prefixSizeBefore = new int[8];
+    private int[] prefixValueCountBefore = new int[8];
     private QwpTableBuffer.ColumnBuffer[] rowFillColumns = new QwpTableBuffer.ColumnBuffer[8];
+    private int[] rowFillColumnPositions = new int[8];
+    private int[] stagedColumnMarks = new int[8];
+    private int currentRowMark = 1;
     private int rowFillColumnCount;
     private int inProgressColumnCount;
 
@@ -198,7 +192,6 @@ public void close() {
             payloadWriter.close();
             datagramSegments.close();
             headerBuffer.close();
-            stagedRow.close();
         }
     }
 
@@ -474,6 +467,7 @@ public Sender table(CharSequence tableName) {
             currentTableBuffer = new QwpTableBuffer(currentTableName);
             tableBuffers.put(currentTableName, currentTableBuffer);
         }
+        rebuildPendingFillColumnsForCurrentTable();
         return this;
     }
 
@@ -516,15 +510,7 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type,
             // and start from scratch with the new schema
 
             if (hasInProgressRow()) {
-                // the slowest case: we are adding a new column while the in-progress row already has some columns.
-                // we need to store the in-progress row in a temporary buffer, flush the committed rows, and then replay the staged row into the new schema
-                // assumption: this case is rare
-
-                snapshotCurrentRowIntoReplayBuffer();
-                rollbackCurrentRowToCommittedState(false);
-                flushCommittedRowsOfCurrentTable();
-                replaySnapshotAsInProgressRow();
-                stagedRow.clear();
+                flushCommittedPrefixPreservingCurrentRow();
             } else {
                 flushCommittedRowsOfCurrentTable();
             }
@@ -538,14 +524,18 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type,
     }
 
     private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence columnName) {
-        InProgressColumnState existing = findInProgressColumnState(column);
-        if (existing != null) {
+        int columnIndex = column.getIndex();
+        ensureStagedColumnMarkCapacity(columnIndex + 1);
+        ensureRowFillPositionCapacity(columnIndex + 1);
+        if (stagedColumnMarks[columnIndex] == currentRowMark) {
             if (columnName != null && columnName.isEmpty()) {
                 throw new LineSenderException("designated timestamp already set for current row");
             }
             throw new LineSenderException("column '" + columnName + "' already set for current row");
         }
+        stagedColumnMarks[columnIndex] = currentRowMark;
         appendInProgressColumnState(column);
+        removePendingFillColumn(column);
     }
 
     private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) {
@@ -559,16 +549,6 @@ private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) {
         inProgressColumnCount++;
     }
 
-    private InProgressColumnState findInProgressColumnState(QwpTableBuffer.ColumnBuffer column) {
-        for (int i = 0; i < inProgressColumnCount; i++) {
-            InProgressColumnState state = inProgressColumns[i];
-            if (state.column == column) {
-                return state;
-            }
-        }
-        return null;
-    }
-
     private void stageBooleanColumnValue(CharSequence name, boolean value) {
         QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false);
         beginColumnWrite(col, name);
@@ -758,39 +738,24 @@ private void clearInProgressRow() {
                 state.clear();
             }
         }
+        if (inProgressColumnCount > 0) {
+            currentRowMark++;
+            if (currentRowMark == 0) {
+                Arrays.fill(stagedColumnMarks, 0);
+                currentRowMark = 1;
+            }
+        }
         inProgressColumnCount = 0;
-        rowFillColumnCount = 0;
     }
 
     private void rollbackCurrentRowToCommittedState() {
-        rollbackCurrentRowToCommittedState(true);
-    }
-
-    private void rollbackCurrentRowToCommittedState(boolean clearReplayBuffer) {
         if (currentTableBuffer != null) {
             currentTableBuffer.cancelCurrentRow();
             currentTableBuffer.rollbackUncommittedColumns();
         }
+        restorePendingFillColumnsFromInProgressColumns();
         clearCachedTimestampColumns();
         clearInProgressRow();
-        if (clearReplayBuffer) {
-            clearReplayBuffer();
-        }
-    }
-
-    private void collectRowFillColumns(int targetRows) {
-        rowFillColumnCount = 0;
-        for (int i = 0, n = currentTableBuffer.getColumnCount(); i < n; i++) {
-            QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i);
-            if (isStagedColumn(col)) {
-                continue;
-            }
-            if (col.getSize() >= targetRows) {
-                continue;
-            }
-            ensureRowFillColumnCapacity(rowFillColumnCount + 1);
-            rowFillColumns[rowFillColumnCount++] = col;
-        }
     }
 
     private void commitCurrentRow() {
@@ -800,20 +765,14 @@ private void commitCurrentRow() {
 
         long estimate = 0;
         int targetRows = currentTableBuffer.getRowCount() + 1;
-        collectRowFillColumns(targetRows);
         if (trackDatagramEstimate) {
             estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows);
             if (estimate > maxDatagramSize) {
                 if (currentTableBuffer.getRowCount() == 0) {
                     throw singleRowTooLarge(estimate);
                 }
-                snapshotCurrentRowIntoReplayBuffer();
-                rollbackCurrentRowToCommittedState(false);
-                flushCommittedRowsOfCurrentTable();
-                replaySnapshotAsInProgressRow();
-                stagedRow.clear();
+                flushCommittedPrefixPreservingCurrentRow();
                 targetRows = currentTableBuffer.getRowCount() + 1;
-                collectRowFillColumns(targetRows);
                 estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows);
                 if (estimate > maxDatagramSize) {
                     throw singleRowTooLarge(estimate);
@@ -822,6 +781,7 @@ private void commitCurrentRow() {
         }
 
         currentTableBuffer.nextRow(rowFillColumns, rowFillColumnCount);
+        restorePendingFillColumnsFromInProgressColumns();
         if (trackDatagramEstimate) {
             committedDatagramEstimate = estimate;
         }
@@ -845,6 +805,23 @@ private int encodeTablePayloadForUdp(QwpTableBuffer tableBuffer) {
         return payloadWriter.getPosition();
     }
 
+    private int encodeCommittedPrefixPayloadForUdp(QwpTableBuffer tableBuffer) {
+        payloadWriter.reset();
+        columnWriter.setBuffer(payloadWriter);
+        columnWriter.encodeTable(
+                tableBuffer,
+                tableBuffer.getRowCount(),
+                prefixValueCountBefore,
+                prefixStringDataSizeBefore,
+                prefixSymbolDictionarySizeBefore,
+                false,
+                false,
+                false
+        );
+        payloadWriter.finish();
+        return payloadWriter.getPosition();
+    }
+
     private long estimateBaseForCurrentSchema() {
         long estimate = HEADER_SIZE;
         int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName);
@@ -930,6 +907,21 @@ private void ensureRowFillColumnCapacity(int required) {
         rowFillColumns = newArr;
     }
 
+    private void ensureRowFillPositionCapacity(int required) {
+        if (required <= rowFillColumnPositions.length) {
+            return;
+        }
+
+        int newCapacity = rowFillColumnPositions.length;
+        while (newCapacity < required) {
+            newCapacity *= 2;
+        }
+
+        int[] newArr = new int[newCapacity];
+        System.arraycopy(rowFillColumnPositions, 0, newArr, 0, rowFillColumnPositions.length);
+        rowFillColumnPositions = newArr;
+    }
+
     private void ensureInProgressColumnCapacity(int required) {
         if (required <= inProgressColumns.length) {
             return;
@@ -945,6 +937,21 @@ private void ensureInProgressColumnCapacity(int required) {
         inProgressColumns = newArr;
     }
 
+    private void ensureStagedColumnMarkCapacity(int required) {
+        if (required <= stagedColumnMarks.length) {
+            return;
+        }
+
+        int newCapacity = stagedColumnMarks.length;
+        while (newCapacity < required) {
+            newCapacity *= 2;
+        }
+
+        int[] newArr = new int[newCapacity];
+        System.arraycopy(stagedColumnMarks, 0, newArr, 0, stagedColumnMarks.length);
+        stagedColumnMarks = newArr;
+    }
+
     private long estimateArrayPayloadBytes(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) {
         int shapeCount = col.getArrayShapeOffset() - state.arrayShapeOffsetBefore;
         int dataCount = col.getArrayDataOffset() - state.arrayDataOffsetBefore;
@@ -990,8 +997,9 @@ private void flushCommittedRowsOfCurrentTable() {
         if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0) {
             return;
         }
-        sendTableBuffer(currentTableName, currentTableBuffer);
+        sendWholeTableBuffer(currentTableName, currentTableBuffer);
         clearCachedTimestampColumns();
+        rebuildPendingFillColumnsForCurrentTable();
         resetCommittedDatagramEstimate();
     }
 
@@ -1006,14 +1014,15 @@ private void flushInternal() {
             if (tableBuffer == null || tableBuffer.getRowCount() == 0) {
                 continue;
             }
-            sendTableBuffer(tableName, tableBuffer);
+            sendWholeTableBuffer(tableName, tableBuffer);
         }
         clearTransientRowState();
+        rebuildPendingFillColumnsForCurrentTable();
         resetCommittedDatagramEstimate();
     }
 
     private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) {
-        sendTableBuffer(tableName, tableBuffer);
+        sendWholeTableBuffer(tableName, tableBuffer);
         clearTransientRowState();
         resetCommittedDatagramEstimate();
     }
@@ -1022,56 +1031,6 @@ private boolean hasInProgressRow() {
         return inProgressColumnCount > 0;
     }
 
-    private boolean isStagedColumn(QwpTableBuffer.ColumnBuffer col) {
-        for (int i = 0; i < inProgressColumnCount; i++) {
-            InProgressColumnState state = inProgressColumns[i];
-            if (state != null && state.column == col) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private void snapshotCurrentRowIntoReplayBuffer() {
-        stagedRow.clear();
-        for (int i = 0; i < inProgressColumnCount; i++) {
-            InProgressColumnState state = inProgressColumns[i];
-            int replayKind;
-            if (state.column == cachedTimestampColumn) {
-                replayKind = ENTRY_AT_MICROS;
-            } else if (state.column == cachedTimestampNanosColumn) {
-                replayKind = ENTRY_AT_NANOS;
-            } else {
-                replayKind = 0;
-            }
-            stagedRow.snapshotInProgressColumn(state, replayKind);
-        }
-    }
-
-    private void replaySnapshotAsInProgressRow() {
-        clearInProgressRow();
-        cachedTimestampColumn = null;
-        cachedTimestampNanosColumn = null;
-        for (int i = 0, n = stagedRow.size(); i < n; i++) {
-            QwpTableBuffer.ColumnBuffer column = currentTableBuffer.getOrCreateColumn(
-                    stagedRow.getColumnName(i),
-                    stagedRow.getColumnType(i),
-                    stagedRow.isColumnNullable(i)
-            );
-            appendInProgressColumnState(column);
-            stagedRow.appendEntryIntoColumn(i, column);
-
-            int kind = stagedRow.getKind(i);
-            if (kind == ENTRY_AT_MICROS) {
-                cachedTimestampColumn = column;
-            } else if (kind == ENTRY_AT_NANOS) {
-                cachedTimestampNanosColumn = column;
-            } else {
-                inProgressRowValueCount++;
-            }
-        }
-    }
-
     private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) {
         return switch (type) {
             case TYPE_BOOLEAN -> packedBytes(valuesBefore + missing) - packedBytes(valuesBefore);
@@ -1104,14 +1063,10 @@ private void clearCachedTimestampColumns() {
         cachedTimestampNanosColumn = null;
     }
 
-    private void clearReplayBuffer() {
-        stagedRow.clear();
-    }
-
     private void clearTransientRowState() {
         clearCachedTimestampColumns();
         clearInProgressRow();
-        clearReplayBuffer();
+        clearPendingFillColumns();
     }
 
     // Public test hooks because module boundaries prevent tests from sharing this package.
@@ -1130,8 +1085,142 @@ public QwpTableBuffer currentTableBufferForTest() {
         return currentTableBuffer;
     }
 
-    private void sendTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer) {
-        int payloadLength = encodeTablePayloadForUdp(tableBuffer);
+    private void captureInProgressColumnPrefixState() {
+        int columnCount = currentTableBuffer.getColumnCount();
+        ensurePrefixColumnCapacity(columnCount);
+        for (int i = 0; i < columnCount; i++) {
+            prefixSizeBefore[i] = -1;
+            prefixValueCountBefore[i] = -1;
+        }
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            InProgressColumnState state = inProgressColumns[i];
+            int columnIndex = state.column.getIndex();
+            prefixSizeBefore[columnIndex] = state.sizeBefore;
+            prefixValueCountBefore[columnIndex] = state.valueCountBefore;
+            prefixStringDataSizeBefore[columnIndex] = state.stringDataSizeBefore;
+            prefixArrayShapeOffsetBefore[columnIndex] = state.arrayShapeOffsetBefore;
+            prefixArrayDataOffsetBefore[columnIndex] = state.arrayDataOffsetBefore;
+            prefixSymbolDictionarySizeBefore[columnIndex] = state.symbolDictionarySizeBefore;
+        }
+    }
+
+    private void ensurePrefixColumnCapacity(int required) {
+        if (required <= prefixSizeBefore.length) {
+            return;
+        }
+
+        int newCapacity = prefixSizeBefore.length;
+        while (newCapacity < required) {
+            newCapacity *= 2;
+        }
+
+        prefixSizeBefore = new int[newCapacity];
+        prefixValueCountBefore = new int[newCapacity];
+        prefixStringDataSizeBefore = new long[newCapacity];
+        prefixArrayShapeOffsetBefore = new int[newCapacity];
+        prefixArrayDataOffsetBefore = new int[newCapacity];
+        prefixSymbolDictionarySizeBefore = new int[newCapacity];
+    }
+
+    private void flushCommittedPrefixPreservingCurrentRow() {
+        if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0) {
+            return;
+        }
+
+        captureInProgressColumnPrefixState();
+        sendCommittedPrefix(currentTableName, currentTableBuffer);
+        currentTableBuffer.retainInProgressRow(
+                prefixSizeBefore,
+                prefixValueCountBefore,
+                prefixStringDataSizeBefore,
+                prefixArrayShapeOffsetBefore,
+                prefixArrayDataOffsetBefore
+        );
+        resetCommittedDatagramEstimate();
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            inProgressColumns[i].rebaseToEmptyTable();
+        }
+    }
+
+    private void clearPendingFillColumns() {
+        for (int i = 0; i < rowFillColumnCount; i++) {
+            QwpTableBuffer.ColumnBuffer column = rowFillColumns[i];
+            if (column != null) {
+                rowFillColumnPositions[column.getIndex()] = 0;
+                rowFillColumns[i] = null;
+            }
+        }
+        rowFillColumnCount = 0;
+    }
+
+    private void rebuildPendingFillColumnsForCurrentTable() {
+        clearPendingFillColumns();
+        if (currentTableBuffer == null) {
+            return;
+        }
+
+        int columnCount = currentTableBuffer.getColumnCount();
+        ensureRowFillColumnCapacity(columnCount);
+        ensureRowFillPositionCapacity(columnCount);
+        for (int i = 0; i < columnCount; i++) {
+            QwpTableBuffer.ColumnBuffer column = currentTableBuffer.getColumn(i);
+            rowFillColumns[rowFillColumnCount] = column;
+            rowFillColumnPositions[i] = rowFillColumnCount + 1;
+            rowFillColumnCount++;
+        }
+    }
+
+    private void removePendingFillColumn(QwpTableBuffer.ColumnBuffer column) {
+        int columnIndex = column.getIndex();
+        if (columnIndex >= rowFillColumnPositions.length) {
+            return;
+        }
+
+        int position = rowFillColumnPositions[columnIndex] - 1;
+        if (position < 0) {
+            return;
+        }
+
+        int lastIndex = rowFillColumnCount - 1;
+        QwpTableBuffer.ColumnBuffer lastColumn = rowFillColumns[lastIndex];
+        rowFillColumns[lastIndex] = null;
+        rowFillColumnCount = lastIndex;
+        rowFillColumnPositions[columnIndex] = 0;
+        if (position != lastIndex) {
+            rowFillColumns[position] = lastColumn;
+            rowFillColumnPositions[lastColumn.getIndex()] = position + 1;
+        }
+    }
+
+    private void restorePendingFillColumnsFromInProgressColumns() {
+        if (currentTableBuffer == null) {
+            return;
+        }
+
+        int columnCount = currentTableBuffer.getColumnCount();
+        ensureRowFillColumnCapacity(rowFillColumnCount + inProgressColumnCount);
+        ensureRowFillPositionCapacity(columnCount);
+        for (int i = 0; i < inProgressColumnCount; i++) {
+            QwpTableBuffer.ColumnBuffer column = inProgressColumns[i].column;
+            int columnIndex = column.getIndex();
+            if (columnIndex >= columnCount || currentTableBuffer.getColumn(columnIndex) != column) {
+                continue;
+            }
+            if (rowFillColumnPositions[columnIndex] != 0) {
+                continue;
+            }
+            rowFillColumns[rowFillColumnCount] = column;
+            rowFillColumnPositions[columnIndex] = rowFillColumnCount + 1;
+            rowFillColumnCount++;
+        }
+    }
+
+    private void sendCommittedPrefix(CharSequence tableName, QwpTableBuffer tableBuffer) {
+        int payloadLength = encodeCommittedPrefixPayloadForUdp(tableBuffer);
+        sendEncodedPayload(tableName, payloadLength);
+    }
+
+    private void sendEncodedPayload(CharSequence tableName, int payloadLength) {
         headerBuffer.reset();
         headerBuffer.putByte((byte) 'Q');
         headerBuffer.putByte((byte) 'W');
@@ -1155,6 +1244,11 @@ private void sendTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer)
         } catch (LineSenderException e) {
             LOG.warn("UDP send failed [table={}, errno={}]: {}", tableName, channel.errno(), String.valueOf(e));
         }
+    }
+
+    private void sendWholeTableBuffer(CharSequence tableName, QwpTableBuffer tableBuffer) {
+        int payloadLength = encodeTablePayloadForUdp(tableBuffer);
+        sendEncodedPayload(tableName, payloadLength);
         tableBuffer.reset();
     }
 
@@ -1227,333 +1321,14 @@ void of(QwpTableBuffer.ColumnBuffer column) {
             this.arrayDataOffsetBefore = column.getArrayDataOffset();
             this.symbolDictionarySizeBefore = column.getSymbolDictionarySize();
         }
-    }
-
-    private static final class NativeRowStaging {
-        private static final int AUX_INT_OFFSET = 4;
-        private static final int ENTRY_SIZE = 40;
-        private static final int LONG0_OFFSET = 8;
-        private static final int LONG1_OFFSET = 16;
-        private static final int LONG2_OFFSET = 24;
-        private static final int LONG3_OFFSET = 32;
-
-        private final Decimal128 decimal128Sink = new Decimal128();
-        private final Decimal256 decimal256Sink = new Decimal256();
-        private final Decimal64 decimal64Sink = new Decimal64();
-        private final OffHeapAppendMemory entries = new OffHeapAppendMemory(ENTRY_SIZE * 8L);
-        private final OffHeapAppendMemory varData = new OffHeapAppendMemory(128);
-        private String[] columnNames = new String[8];
-        private byte[] columnTypes = new byte[8];
-        private boolean[] columnNullables = new boolean[8];
-        private int size;
-
-        void appendEntryIntoColumn(int index, QwpTableBuffer.ColumnBuffer column) {
-            long entryAddress = entryAddress(index);
-            int kind = Unsafe.getUnsafe().getInt(entryAddress);
-            int auxInt = Unsafe.getUnsafe().getInt(entryAddress + AUX_INT_OFFSET);
-            long long0 = Unsafe.getUnsafe().getLong(entryAddress + LONG0_OFFSET);
-            long long1 = Unsafe.getUnsafe().getLong(entryAddress + LONG1_OFFSET);
-            switch (kind) {
-                case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS ->
-                        column.addLong(long0);
-                case ENTRY_BOOL -> column.addBoolean(long0 != 0);
-                case ENTRY_DECIMAL64 -> {
-                    decimal64Sink.ofRaw(long0);
-                    decimal64Sink.setScale(auxInt);
-                    column.addDecimal64(decimal64Sink);
-                }
-                case ENTRY_DECIMAL128 -> {
-                    decimal128Sink.ofRaw(long0, long1);
-                    decimal128Sink.setScale(auxInt);
-                    column.addDecimal128(decimal128Sink);
-                }
-                case ENTRY_DECIMAL256 -> {
-                    decimal256Sink.ofRaw(
-                            long0,
-                            long1,
-                            Unsafe.getUnsafe().getLong(entryAddress + LONG2_OFFSET),
-                            Unsafe.getUnsafe().getLong(entryAddress + LONG3_OFFSET)
-                    );
-                    decimal256Sink.setScale(auxInt);
-                    column.addDecimal256(decimal256Sink);
-                }
-                case ENTRY_DOUBLE -> column.addDouble(Double.longBitsToDouble(long0));
-                case ENTRY_DOUBLE_ARRAY -> column.addDoubleArrayPayload(varData.addressOf(long1), long0);
-                case ENTRY_LONG_ARRAY -> column.addLongArrayPayload(varData.addressOf(long1), long0);
-                case ENTRY_STRING -> column.addStringUtf8(varData.addressOf(long0), auxInt);
-                case ENTRY_SYMBOL -> {
-                    if (auxInt < 0) {
-                        column.addSymbol(null);
-                    } else {
-                        column.addSymbolUtf8(varData.addressOf(long0), auxInt);
-                    }
-                }
-                default -> throw new LineSenderException("unknown staged row entry type: " + kind);
-            }
-        }
-
-        void clear() {
-            for (int i = 0; i < size; i++) {
-                columnNames[i] = null;
-            }
-            size = 0;
-            entries.truncate();
-            varData.truncate();
-        }
-
-        void close() {
-            entries.close();
-            varData.close();
-        }
-
-        byte getColumnType(int index) {
-            return columnTypes[index];
-        }
-
-        String getColumnName(int index) {
-            return columnNames[index];
-        }
-
-        int getKind(int index) {
-            return Unsafe.getUnsafe().getInt(entryAddress(index));
-        }
-
-        boolean isColumnNullable(int index) {
-            return columnNullables[index];
-        }
 
-        void snapshotInProgressColumn(InProgressColumnState state, int replayKind) {
-            QwpTableBuffer.ColumnBuffer column = state.column;
-            int kind = replayKind != 0 ? replayKind : defaultKind(column.getType());
-            int valueIndex = state.valueCountBefore;
-            long dataAddress = column.getDataAddress();
-
-            switch (kind) {
-                case ENTRY_AT_MICROS, ENTRY_AT_NANOS, ENTRY_LONG, ENTRY_TIMESTAMP_COL_MICROS, ENTRY_TIMESTAMP_COL_NANOS -> {
-                    long value = Unsafe.getUnsafe().getLong(dataAddress + (long) valueIndex * Long.BYTES);
-                    stageEntry(column, kind, 0, value, 0, 0, 0);
-                }
-                case ENTRY_BOOL -> {
-                    boolean value = Unsafe.getUnsafe().getByte(dataAddress + valueIndex) != 0;
-                    stageEntry(column, ENTRY_BOOL, 0, value ? 1 : 0, 0, 0, 0);
-                }
-                case ENTRY_DECIMAL64 -> {
-                    long value = Unsafe.getUnsafe().getLong(dataAddress + (long) valueIndex * Long.BYTES);
-                    stageEntry(column, ENTRY_DECIMAL64, column.getDecimalScale(), value, 0, 0, 0);
-                }
-                case ENTRY_DECIMAL128 -> {
-                    long offset = (long) valueIndex * 16;
-                    stageEntry(
-                            column,
-                            ENTRY_DECIMAL128,
-                            column.getDecimalScale(),
-                            Unsafe.getUnsafe().getLong(dataAddress + offset),
-                            Unsafe.getUnsafe().getLong(dataAddress + offset + Long.BYTES),
-                            0,
-                            0
-                    );
-                }
-                case ENTRY_DECIMAL256 -> {
-                    long offset = (long) valueIndex * 32;
-                    stageEntry(
-                            column,
-                            ENTRY_DECIMAL256,
-                            column.getDecimalScale(),
-                            Unsafe.getUnsafe().getLong(dataAddress + offset),
-                            Unsafe.getUnsafe().getLong(dataAddress + offset + 8),
-                            Unsafe.getUnsafe().getLong(dataAddress + offset + 16),
-                            Unsafe.getUnsafe().getLong(dataAddress + offset + 24)
-                    );
-                }
-                case ENTRY_DOUBLE -> {
-                    long value = Unsafe.getUnsafe().getLong(dataAddress + (long) valueIndex * Double.BYTES);
-                    stageEntry(column, ENTRY_DOUBLE, 0, value, 0, 0, 0);
-                }
-                case ENTRY_DOUBLE_ARRAY -> snapshotDoubleArray(column, state);
-                case ENTRY_LONG_ARRAY -> snapshotLongArray(column, state);
-                case ENTRY_STRING -> snapshotString(column, state);
-                case ENTRY_SYMBOL -> snapshotSymbol(column, state);
-                default -> throw new LineSenderException("unsupported replay row entry type: " + kind);
-            }
-        }
-
-        int size() {
-            return size;
-        }
-
-        private int defaultKind(byte columnType) {
-            return switch (columnType) {
-                case TYPE_BOOLEAN -> ENTRY_BOOL;
-                case TYPE_DECIMAL128 -> ENTRY_DECIMAL128;
-                case TYPE_DECIMAL256 -> ENTRY_DECIMAL256;
-                case TYPE_DECIMAL64 -> ENTRY_DECIMAL64;
-                case TYPE_DOUBLE -> ENTRY_DOUBLE;
-                case TYPE_DOUBLE_ARRAY -> ENTRY_DOUBLE_ARRAY;
-                case TYPE_LONG -> ENTRY_LONG;
-                case TYPE_LONG_ARRAY -> ENTRY_LONG_ARRAY;
-                case TYPE_STRING, TYPE_VARCHAR -> ENTRY_STRING;
-                case TYPE_SYMBOL -> ENTRY_SYMBOL;
-                case TYPE_TIMESTAMP -> ENTRY_TIMESTAMP_COL_MICROS;
-                case TYPE_TIMESTAMP_NANOS -> ENTRY_TIMESTAMP_COL_NANOS;
-                default -> throw new LineSenderException("unsupported replay row column type: " + columnType);
-            };
-        }
-
-        private long entryAddress(int index) {
-            return entries.addressOf((long) index * ENTRY_SIZE);
-        }
-
-        private void ensureCapacity(int required) {
-            if (required <= columnNames.length) {
-                return;
-            }
-
-            int newCapacity = columnNames.length;
-            while (newCapacity < required) {
-                newCapacity *= 2;
-            }
-
-            String[] newColumnNames = new String[newCapacity];
-            System.arraycopy(columnNames, 0, newColumnNames, 0, size);
-            columnNames = newColumnNames;
-
-            byte[] newColumnTypes = new byte[newCapacity];
-            System.arraycopy(columnTypes, 0, newColumnTypes, 0, size);
-            columnTypes = newColumnTypes;
-
-            boolean[] newColumnNullables = new boolean[newCapacity];
-            System.arraycopy(columnNullables, 0, newColumnNullables, 0, size);
-            columnNullables = newColumnNullables;
-        }
-
-        private void snapshotDoubleArray(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
-            if (column.getValueCount() == state.valueCountBefore) {
-                stageEntry(column, ENTRY_DOUBLE_ARRAY, 0, -1, 0, 0, 0);
-                return;
-            }
-
-            long offset = varData.getAppendOffset();
-            try {
-                int shapeStart = state.arrayShapeOffsetBefore;
-                int shapeEnd = column.getArrayShapeOffset();
-                int dataStart = state.arrayDataOffsetBefore;
-                int dataEnd = column.getArrayDataOffset();
-                int[] shapes = column.getArrayShapes();
-                double[] data = column.getDoubleArrayData();
-
-                varData.putByte((byte) (shapeEnd - shapeStart));
-                for (int i = shapeStart; i < shapeEnd; i++) {
-                    varData.putInt(shapes[i]);
-                }
-                for (int i = dataStart; i < dataEnd; i++) {
-                    varData.putDouble(data[i]);
-                }
-
-                long payloadLength = varData.getAppendOffset() - offset;
-                stageEntry(column, ENTRY_DOUBLE_ARRAY, 0, payloadLength, offset, 0, 0);
-            } catch (Throwable t) {
-                varData.jumpTo(offset);
-                throw t;
-            }
-        }
-
-        private void snapshotLongArray(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
-            if (column.getValueCount() == state.valueCountBefore) {
-                stageEntry(column, ENTRY_LONG_ARRAY, 0, -1, 0, 0, 0);
-                return;
-            }
-
-            long offset = varData.getAppendOffset();
-            try {
-                int shapeStart = state.arrayShapeOffsetBefore;
-                int shapeEnd = column.getArrayShapeOffset();
-                int dataStart = state.arrayDataOffsetBefore;
-                int dataEnd = column.getArrayDataOffset();
-                int[] shapes = column.getArrayShapes();
-                long[] data = column.getLongArrayData();
-
-                varData.putByte((byte) (shapeEnd - shapeStart));
-                for (int i = shapeStart; i < shapeEnd; i++) {
-                    varData.putInt(shapes[i]);
-                }
-                for (int i = dataStart; i < dataEnd; i++) {
-                    varData.putLong(data[i]);
-                }
-
-                long payloadLength = varData.getAppendOffset() - offset;
-                stageEntry(column, ENTRY_LONG_ARRAY, 0, payloadLength, offset, 0, 0);
-            } catch (Throwable t) {
-                varData.jumpTo(offset);
-                throw t;
-            }
-        }
-
-        private void snapshotString(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
-            if (column.getValueCount() == state.valueCountBefore) {
-                stageEntry(column, ENTRY_STRING, -1, 0, 0, 0, 0);
-                return;
-            }
-
-            long offsetsAddress = column.getStringOffsetsAddress();
-            int start = Unsafe.getUnsafe().getInt(offsetsAddress + (long) state.valueCountBefore * Integer.BYTES);
-            int end = Unsafe.getUnsafe().getInt(offsetsAddress + (long) (state.valueCountBefore + 1) * Integer.BYTES);
-            int len = end - start;
-
-            long offset = varData.getAppendOffset();
-            try {
-                if (len > 0) {
-                    varData.putBlockOfBytes(column.getStringDataAddress() + start, len);
-                }
-                stageEntry(column, ENTRY_STRING, len, offset, 0, 0, 0);
-            } catch (Throwable t) {
-                varData.jumpTo(offset);
-                throw t;
-            }
-        }
-
-        private void snapshotSymbol(QwpTableBuffer.ColumnBuffer column, InProgressColumnState state) {
-            if (column.getValueCount() == state.valueCountBefore) {
-                stageEntry(column, ENTRY_SYMBOL, -1, 0, 0, 0, 0);
-                return;
-            }
-
-            int valueIndex = state.valueCountBefore;
-            int localIndex = Unsafe.getUnsafe().getInt(column.getDataAddress() + (long) valueIndex * Integer.BYTES);
-            CharSequence value = column.getSymbolValue(localIndex);
-            stageUtf8(column, ENTRY_SYMBOL, value);
-        }
-
-        private void stageEntry(
-                QwpTableBuffer.ColumnBuffer column,
-                int kind,
-                int auxInt,
-                long long0,
-                long long1,
-                long long2,
-                long long3
-        ) {
-            ensureCapacity(size + 1);
-            columnNames[size] = column.getName();
-            columnTypes[size] = column.getType();
-            columnNullables[size] = column.isNullable();
-            entries.putInt(kind);
-            entries.putInt(auxInt);
-            entries.putLong(long0);
-            entries.putLong(long1);
-            entries.putLong(long2);
-            entries.putLong(long3);
-            size++;
-        }
-
-        private void stageUtf8(QwpTableBuffer.ColumnBuffer column, int kind, CharSequence value) {
-            int len = -1;
-            long offset = 0;
-            if (value != null) {
-                offset = varData.getAppendOffset();
-                varData.putUtf8(value);
-                len = (int) (varData.getAppendOffset() - offset);
-            }
-            stageEntry(column, kind, len, offset, 0, 0, 0);
+        void rebaseToEmptyTable() {
+            this.sizeBefore = 0;
+            this.valueCountBefore = 0;
+            this.stringDataSizeBefore = 0;
+            this.arrayShapeOffsetBefore = 0;
+            this.arrayDataOffsetBefore = 0;
+            this.symbolDictionarySizeBefore = 0;
         }
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index d6366cb..e4c46f4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -195,6 +195,7 @@ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean null
     private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) {
         ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable);
         int index = columns.size();
+        col.index = index;
         columns.add(col);
         columnNameToIndex.put(name, index);
         // Update fast access array
@@ -240,6 +241,7 @@ private void rebuildColumnAccessStructures() {
 
         for (int i = 0; i < columnCount; i++) {
             ColumnBuffer col = columns.getQuick(i);
+            col.index = i;
             fastColumns[i] = col;
             columnNameToIndex.put(col.name, i);
         }
@@ -358,6 +360,32 @@ public void nextRow(ColumnBuffer[] missingColumns, int missingColumnCount) {
         committedColumnCount = columns.size();
     }
 
+    public void retainInProgressRow(
+            int[] sizeBefore,
+            int[] valueCountBefore,
+            long[] stringDataSizeBefore,
+            int[] arrayShapeOffsetBefore,
+            int[] arrayDataOffsetBefore
+    ) {
+        columnAccessCursor = 0;
+        for (int i = 0, n = columns.size(); i < n; i++) {
+            ColumnBuffer col = fastColumns[i];
+            if (sizeBefore[i] > -1) {
+                col.retainTailRow(
+                        sizeBefore[i],
+                        valueCountBefore[i],
+                        stringDataSizeBefore[i],
+                        arrayShapeOffsetBefore[i],
+                        arrayDataOffsetBefore[i]
+                );
+            } else {
+                col.clearToEmptyFast();
+            }
+        }
+        rowCount = 0;
+        committedColumnCount = columns.size();
+    }
+
     /**
      * Resets the buffer for reuse. Keeps column definitions and allocated memory.
      */
@@ -512,6 +540,7 @@ public static class ColumnBuffer implements QuietCloseable {
         // GeoHash precision (number of bits, 1-60)
         private int geohashPrecision = -1;
         private boolean hasNulls;
+        private int index;
         private long[] longArrayData;
         private int maxGlobalSymbolId = -1;
         private int nullBufCapRows;
@@ -1057,6 +1086,10 @@ public void addSymbolWithGlobalId(String value, int globalId) {
             size++;
         }
 
+        public int getIndex() {
+            return index;
+        }
+
         private int getOrAddLocalSymbol(CharSequence value) {
             int idx = symbolDict.get(value);
             if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
@@ -1282,6 +1315,36 @@ public void reset() {
             geohashPrecision = -1;
         }
 
+        public void retainTailRow(
+                int sizeBefore,
+                int valueCountBefore,
+                long stringDataSizeBefore,
+                int arrayShapeOffsetBefore,
+                int arrayDataOffsetBefore
+        ) {
+            assert size == sizeBefore + 1;
+
+            compactNullBitmap(sizeBefore);
+
+            if (valueCount == valueCountBefore) {
+                clearValuePayload();
+                size = 1;
+                valueCount = 0;
+                return;
+            }
+
+            switch (type) {
+                case TYPE_STRING, TYPE_VARCHAR -> retainStringValue(valueCountBefore);
+                case TYPE_SYMBOL -> retainSymbolValue(valueCountBefore);
+                case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY ->
+                        retainArrayValue(valueCountBefore, arrayShapeOffsetBefore, arrayDataOffsetBefore);
+                default -> retainFixedWidthValue(valueCountBefore);
+            }
+
+            size = 1;
+            valueCount = 1;
+        }
+
         public void truncateTo(int newSize) {
             if (newSize >= size) {
                 return;
@@ -1367,6 +1430,51 @@ private static int checkedElementCount(long product) {
             return (int) product;
         }
 
+        private void clearValuePayload() {
+            if (dataBuffer != null && elemSize > 0) {
+                dataBuffer.jumpTo(0);
+            }
+            if (auxBuffer != null) {
+                auxBuffer.truncate();
+            }
+            if (stringOffsets != null) {
+                stringOffsets.truncate();
+                stringOffsets.putInt(0);
+            }
+            if (stringData != null) {
+                stringData.truncate();
+            }
+            arrayShapeOffset = 0;
+            arrayDataOffset = 0;
+            resetEmptyMetadata();
+        }
+
+        private void clearToEmptyFast() {
+            int sizeBefore = size;
+            clearValuePayload();
+            if (nullBufPtr != 0 && sizeBefore > 0) {
+                long usedLongs = ((long) sizeBefore + 63) >>> 6;
+                Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0);
+            }
+            size = 0;
+            valueCount = 0;
+            hasNulls = false;
+        }
+
+        private void compactNullBitmap(int sourceRow) {
+            if (nullBufPtr == 0) {
+                return;
+            }
+
+            boolean retainedNull = isNull(sourceRow);
+            long usedLongs = ((long) size + 63) >>> 6;
+            Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0);
+            if (retainedNull) {
+                Unsafe.getUnsafe().putLong(nullBufPtr, 1L);
+            }
+            hasNulls = retainedNull;
+        }
+
         private void appendArrayPayload(long ptr, long len, boolean forLong) {
             if (len < 0) {
                 addNull();
@@ -1552,5 +1660,88 @@ private void markNull(int index) {
             Unsafe.getUnsafe().putLong(longAddr, current | (1L << bitIndex));
             hasNulls = true;
         }
+
+        private void retainArrayValue(int valueIndex, int shapeOffsetBefore, int dataOffsetBefore) {
+            int nDims = arrayDims[valueIndex] & 0xFF;
+            arrayDims[0] = (byte) nDims;
+
+            int shapeCount = arrayShapeOffset - shapeOffsetBefore;
+            if (shapeCount > 0 && shapeOffsetBefore > 0) {
+                System.arraycopy(arrayShapes, shapeOffsetBefore, arrayShapes, 0, shapeCount);
+            }
+            arrayShapeOffset = shapeCount;
+
+            int dataCount = arrayDataOffset - dataOffsetBefore;
+            if (dataCount > 0 && dataOffsetBefore > 0) {
+                if (type == TYPE_LONG_ARRAY) {
+                    System.arraycopy(longArrayData, dataOffsetBefore, longArrayData, 0, dataCount);
+                } else {
+                    System.arraycopy(doubleArrayData, dataOffsetBefore, doubleArrayData, 0, dataCount);
+                }
+            }
+            arrayDataOffset = dataCount;
+        }
+
+        private void retainFixedWidthValue(int valueIndex) {
+            if (dataBuffer == null || elemSize == 0) {
+                return;
+            }
+
+            long srcOffset = (long) valueIndex * elemSize;
+            long dataAddress = dataBuffer.pageAddress();
+            if (srcOffset > 0) {
+                Vect.memmove(dataAddress, dataAddress + srcOffset, elemSize);
+            }
+            dataBuffer.jumpTo(elemSize);
+
+            if (auxBuffer != null) {
+                long auxAddress = auxBuffer.pageAddress();
+                long auxOffset = (long) valueIndex * Integer.BYTES;
+                if (auxOffset > 0) {
+                    Vect.memmove(auxAddress, auxAddress + auxOffset, Integer.BYTES);
+                }
+                auxBuffer.jumpTo(Integer.BYTES);
+                maxGlobalSymbolId = Unsafe.getUnsafe().getInt(auxAddress);
+            }
+        }
+
+        private void retainStringValue(int valueIndex) {
+            long offsetsAddress = stringOffsets.pageAddress();
+            int start = Unsafe.getUnsafe().getInt(offsetsAddress + (long) valueIndex * Integer.BYTES);
+            int end = Unsafe.getUnsafe().getInt(offsetsAddress + (long) (valueIndex + 1) * Integer.BYTES);
+            int len = end - start;
+
+            if (len > 0 && start > 0) {
+                Vect.memmove(stringData.pageAddress(), stringData.pageAddress() + start, len);
+            }
+
+            stringData.jumpTo(len);
+            stringOffsets.truncate();
+            stringOffsets.putInt(0);
+            stringOffsets.putInt(len);
+        }
+
+        private void retainSymbolValue(int valueIndex) {
+            retainFixedWidthValue(valueIndex);
+
+            int localIndex = Unsafe.getUnsafe().getInt(dataBuffer.pageAddress());
+            String symbol = symbolList.get(localIndex);
+
+            symbolDict.clear();
+            symbolList.clear();
+            symbolList.add(symbol);
+            symbolDict.put(symbol, 0);
+            Unsafe.getUnsafe().putInt(dataBuffer.pageAddress(), 0);
+        }
+
+        private void resetEmptyMetadata() {
+            decimalScale = -1;
+            geohashPrecision = -1;
+            maxGlobalSymbolId = -1;
+            if (symbolDict != null) {
+                symbolDict.clear();
+                symbolList.clear();
+            }
+        }
     }
 }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
index 7bf98f9..a525641 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
@@ -139,6 +139,53 @@ public void testBoundedSenderMixedNullablePaddingPreservesRowsAndPacketLimit() t
         });
     }
 
+    @Test
+    public void testBoundedSenderNullableStringNullAcrossOverflowBoundaryPreservesRowsAndPacketLimit() throws Exception {
+        assertMemoryLeak(() -> {
+            String alpha = repeat('a', 512);
+            String marker1 = repeat('m', 64);
+            String marker2 = repeat('n', 64);
+            String marker3 = repeat('o', 64);
+            String omega = repeat('z', 128);
+            List rows = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("x", 1)
+                                    .stringColumn("s", alpha)
+                                    .stringColumn("m", marker1)
+                                    .atNow(),
+                            "x", 1L,
+                            "s", alpha,
+                            "m", marker1),
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("x", 2)
+                                    .stringColumn("s", null)
+                                    .stringColumn("m", marker2)
+                                    .atNow(),
+                            "x", 2L,
+                            "s", null,
+                            "m", marker2),
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("x", 3)
+                                    .stringColumn("s", omega)
+                                    .stringColumn("m", marker3)
+                                    .atNow(),
+                            "x", 3L,
+                            "s", omega,
+                            "m", marker3)
+            );
+            int firstRowPacket = fullPacketSize(rows.subList(0, 1));
+            int twoRowPacket = fullPacketSize(rows.subList(0, 2));
+            int maxDatagramSize = firstRowPacket + 16;
+            Assert.assertTrue("expected overflow boundary between row 1 and row 2", maxDatagramSize < twoRowPacket);
+
+            RunResult result = runScenario(rows, maxDatagramSize);
+
+            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
+            assertPacketsWithinLimit(result, maxDatagramSize);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
     @Test
     public void testUnboundedSenderOmittedNullableAndNonNullableColumnsPreservesRows() throws Exception {
         assertMemoryLeak(() -> {
@@ -172,6 +219,54 @@ public void testUnboundedSenderOmittedNullableAndNonNullableColumnsPreservesRows
         });
     }
 
+    @Test
+    public void testUnboundedSenderWideSchemaWithLowIndexWritePreservesRows() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("wide", sender -> sender.table("wide")
+                                    .longColumn("c0", 0)
+                                    .longColumn("c1", 1)
+                                    .longColumn("c2", 2)
+                                    .longColumn("c3", 3)
+                                    .longColumn("c4", 4)
+                                    .longColumn("c5", 5)
+                                    .longColumn("c6", 6)
+                                    .longColumn("c7", 7)
+                                    .longColumn("c8", 8)
+                                    .longColumn("c9", 9)
+                                    .atNow(),
+                            "c0", 0L,
+                            "c1", 1L,
+                            "c2", 2L,
+                            "c3", 3L,
+                            "c4", 4L,
+                            "c5", 5L,
+                            "c6", 6L,
+                            "c7", 7L,
+                            "c8", 8L,
+                            "c9", 9L),
+                    row("wide", sender -> sender.table("wide")
+                                    .longColumn("c0", 10)
+                                    .atNow(),
+                            "c0", 10L,
+                            "c1", Long.MIN_VALUE,
+                            "c2", Long.MIN_VALUE,
+                            "c3", Long.MIN_VALUE,
+                            "c4", Long.MIN_VALUE,
+                            "c5", Long.MIN_VALUE,
+                            "c6", Long.MIN_VALUE,
+                            "c7", Long.MIN_VALUE,
+                            "c8", Long.MIN_VALUE,
+                            "c9", Long.MIN_VALUE)
+            );
+
+            RunResult result = runScenario(rows, 0);
+
+            Assert.assertEquals(1, result.sendCount);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
     @Test
     public void testBoundedSenderOmittedNonNullableColumnsPreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
@@ -439,6 +534,62 @@ public void testBoundedSenderMixedTypesPreservesRowsAndPacketLimit() throws Exce
         });
     }
 
+    @Test
+    public void testBoundedSenderRepeatedOverflowBoundariesWithDistinctSymbolsPreserveRowsAndPacketLimit() throws Exception {
+        assertMemoryLeak(() -> {
+            String payload = repeat('p', 256);
+            List rows = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v0")
+                                    .longColumn("x", 0)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v0",
+                            "x", 0L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v1")
+                                    .longColumn("x", 1)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v1",
+                            "x", 1L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v2")
+                                    .longColumn("x", 2)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v2",
+                            "x", 2L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v3")
+                                    .longColumn("x", 3)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v3",
+                            "x", 3L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v4")
+                                    .longColumn("x", 4)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v4",
+                            "x", 4L,
+                            "s", payload)
+            );
+            int maxDatagramSize = fullPacketSize(rows.subList(0, 2)) - 1;
+
+            RunResult result = runScenario(rows, maxDatagramSize);
+
+            Assert.assertEquals(rows.size(), result.sendCount);
+            assertPacketsWithinLimit(result, maxDatagramSize);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
     @Test
     public void testBoundedSenderOutOfOrderExistingColumnsPreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
@@ -520,6 +671,30 @@ public void testFlushWhileRowInProgressThrowsAndPreservesRow() throws Exception
         });
     }
 
+    @Test
+    public void testFlushPreservesPendingFillStateForCurrentTable() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .longColumn("b", 2)
+                        .atNow();
+
+                sender.flush();
+
+                sender.longColumn("a", 3).atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L, "b", 2L),
+                    decodedRow("t", "a", 3L, "b", Long.MIN_VALUE)
+            ), decodeRows(nf.packets));
+        });
+    }
+
     @Test
     public void testSimpleLongRowUsesScatterSendPath() throws Exception {
         assertMemoryLeak(() -> {
@@ -893,6 +1068,95 @@ public void testNullableArrayReplayKeepsNullArrayStateWithoutReflection() throws
         });
     }
 
+    @Test
+    public void testNullableStringPrefixFlushKeepsNullStateWithoutReflection() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longColumn("x", 1)
+                        .stringColumn("s", "alpha")
+                        .atNow();
+
+                sender.longColumn("x", 2);
+                sender.stringColumn("s", null);
+                sender.longColumn("b", 3);
+
+                Assert.assertEquals(1, nf.sendCount);
+
+                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
+                Assert.assertNotNull(tableBuffer);
+                Assert.assertEquals(0, tableBuffer.getRowCount());
+
+                QwpTableBuffer.ColumnBuffer stringColumn = tableBuffer.getExistingColumn("s", TYPE_STRING);
+                assertNullableStringNullState(stringColumn);
+
+                QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("x", TYPE_LONG);
+                Assert.assertNotNull(longColumn);
+                Assert.assertEquals(1, longColumn.getSize());
+                Assert.assertEquals(1, longColumn.getValueCount());
+                Assert.assertEquals(2L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress()));
+
+                QwpTableBuffer.ColumnBuffer newColumn = tableBuffer.getExistingColumn("b", TYPE_LONG);
+                Assert.assertNotNull(newColumn);
+                Assert.assertEquals(1, newColumn.getSize());
+                Assert.assertEquals(1, newColumn.getValueCount());
+                Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(newColumn.getDataAddress()));
+
+                sender.atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "x", 1L, "s", "alpha"),
+                    decodedRow("t", "x", 2L, "s", null, "b", 3L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testSymbolPrefixFlushKeepsSingleRetainedDictionaryEntry() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .symbol("sym", "alpha")
+                        .longColumn("x", 1)
+                        .atNow();
+
+                sender.symbol("sym", "beta");
+                sender.longColumn("x", 2);
+                sender.longColumn("b", 3);
+
+                Assert.assertEquals(1, nf.sendCount);
+
+                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
+                Assert.assertNotNull(tableBuffer);
+                Assert.assertEquals(0, tableBuffer.getRowCount());
+
+                QwpTableBuffer.ColumnBuffer symbolColumn = tableBuffer.getExistingColumn("sym", TYPE_SYMBOL);
+                Assert.assertNotNull(symbolColumn);
+                Assert.assertEquals(1, symbolColumn.getSize());
+                Assert.assertEquals(1, symbolColumn.getValueCount());
+                Assert.assertEquals(1, symbolColumn.getSymbolDictionarySize());
+                Assert.assertEquals("beta", symbolColumn.getSymbolValue(0));
+                Assert.assertTrue(symbolColumn.hasSymbol("beta"));
+                Assert.assertFalse(symbolColumn.hasSymbol("alpha"));
+                Assert.assertEquals(0, Unsafe.getUnsafe().getInt(symbolColumn.getDataAddress()));
+
+                sender.atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "sym", "alpha", "x", 1L),
+                    decodedRow("t", "sym", "beta", "x", 2L, "b", 3L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
     @Test
     public void testDuplicateColumnAfterSchemaFlushReplayIsRejected() throws Exception {
         assertMemoryLeak(() -> {
@@ -1398,6 +1662,18 @@ private static void assertNullableArrayNullState(QwpTableBuffer.ColumnBuffer col
         Assert.assertEquals(0, column.getArrayDataOffset());
     }
 
+    private static void assertNullableStringNullState(QwpTableBuffer.ColumnBuffer column) {
+        Assert.assertNotNull(column);
+        Assert.assertEquals(1, column.getSize());
+        Assert.assertEquals(0, column.getValueCount());
+        Assert.assertTrue(column.isNullable());
+        Assert.assertTrue(column.isNull(0));
+        Assert.assertEquals(0, column.getStringDataSize());
+        long offsetsAddress = column.getStringOffsetsAddress();
+        Assert.assertTrue(offsetsAddress != 0);
+        Assert.assertEquals(0, Unsafe.getUnsafe().getInt(offsetsAddress));
+    }
+
     private static BigDecimal decimal(long unscaled, int scale) {
         return BigDecimal.valueOf(unscaled, scale);
     }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
index f7d9e74..dc16bd2 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
@@ -329,6 +329,73 @@ public void testNextRowWithPreparedMissingColumnsPadsListedColumns() throws Exce
         });
     }
 
+    @Test
+    public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exception {
+        assertMemoryLeak(() -> {
+            try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+                QwpTableBuffer.ColumnBuffer keep = table.getOrCreateColumn("keep", QwpConstants.TYPE_LONG, false);
+                QwpTableBuffer.ColumnBuffer drop = table.getOrCreateColumn("drop", QwpConstants.TYPE_STRING, true);
+
+                for (int i = 0; i < 130; i++) {
+                    keep.addLong(i);
+                    if ((i & 1) == 0) {
+                        drop.addString("v" + i);
+                    } else {
+                        drop.addNull();
+                    }
+                    table.nextRow();
+                }
+
+                int keepSizeBefore = keep.getSize();
+                int keepValueCountBefore = keep.getValueCount();
+                long keepStringDataSizeBefore = keep.getStringDataSize();
+                int keepArrayShapeOffsetBefore = keep.getArrayShapeOffset();
+                int keepArrayDataOffsetBefore = keep.getArrayDataOffset();
+                int keepIndex = keep.getIndex();
+
+                keep.addLong(130);
+
+                int[] sizeBefore = {-1, -1};
+                int[] valueCountBefore = {-1, -1};
+                long[] stringDataSizeBefore = new long[2];
+                int[] arrayShapeOffsetBefore = new int[2];
+                int[] arrayDataOffsetBefore = new int[2];
+
+                sizeBefore[keepIndex] = keepSizeBefore;
+                valueCountBefore[keepIndex] = keepValueCountBefore;
+                stringDataSizeBefore[keepIndex] = keepStringDataSizeBefore;
+                arrayShapeOffsetBefore[keepIndex] = keepArrayShapeOffsetBefore;
+                arrayDataOffsetBefore[keepIndex] = keepArrayDataOffsetBefore;
+
+                table.retainInProgressRow(
+                        sizeBefore,
+                        valueCountBefore,
+                        stringDataSizeBefore,
+                        arrayShapeOffsetBefore,
+                        arrayDataOffsetBefore
+                );
+
+                assertEquals(0, table.getRowCount());
+
+                assertEquals(1, keep.getSize());
+                assertEquals(1, keep.getValueCount());
+                assertEquals(130L, Unsafe.getUnsafe().getLong(keep.getDataAddress()));
+
+                assertEquals(0, drop.getSize());
+                assertEquals(0, drop.getValueCount());
+                assertEquals(0, drop.getStringDataSize());
+                assertFalse(drop.isNull(0));
+                assertFalse(drop.isNull(63));
+                assertFalse(drop.isNull(64));
+                assertFalse(drop.isNull(129));
+                assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress()));
+                assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + Long.BYTES));
+                assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + 2L * Long.BYTES));
+                assertEquals(0, Unsafe.getUnsafe().getInt(drop.getStringOffsetsAddress()));
+            }
+        });
+    }
+
     @Test
     public void testCancelRowResetsDecimalScaleOnLateAddedColumn() throws Exception {
         assertMemoryLeak(() -> {

From 6fb1a2568bafcd3619f528f0d60c0b631f00fb18 Mon Sep 17 00:00:00 2001
From: Jaromir Hamala 
Date: Mon, 9 Mar 2026 09:07:22 +0100
Subject: [PATCH 169/230] remove estimator bandaid

---
 .../cutlass/qwp/client/QwpUdpSender.java      | 21 ++++-
 .../cutlass/qwp/client/QwpUdpSenderTest.java  | 93 +++++++++++++++++++
 2 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
index 5cd63bb..c2399e4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
@@ -65,7 +65,6 @@
  */
 public class QwpUdpSender implements Sender {
     private static final int VARINT_INT_UPPER_BOUND = 5;
-    private static final int SAFETY_MARGIN_BYTES = 8;
     private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class);
 
     private final UdpLineChannel channel;
@@ -85,15 +84,24 @@ public class QwpUdpSender implements Sender {
     private int inProgressRowValueCount;
     private QwpTableBuffer currentTableBuffer;
     private String currentTableName;
+
+    // prefix* arrays: per-column snapshots captured before the in-progress row,
+    // used to encode and flush only the committed prefix when a row is still being built.
+    // Indexed by column index. -1 means the column has no in-progress data.
     private int[] prefixArrayDataOffsetBefore = new int[8];
     private int[] prefixArrayShapeOffsetBefore = new int[8];
     private long[] prefixStringDataSizeBefore = new long[8];
     private int[] prefixSymbolDictionarySizeBefore = new int[8];
     private int[] prefixSizeBefore = new int[8];
     private int[] prefixValueCountBefore = new int[8];
+
+    // columns that need NULL/default fill for the current row (columns not yet written to)
     private QwpTableBuffer.ColumnBuffer[] rowFillColumns = new QwpTableBuffer.ColumnBuffer[8];
+    // maps column index -> 1-based position in rowFillColumns (0 means absent)
     private int[] rowFillColumnPositions = new int[8];
+    // per-column marks to detect duplicate writes within a single row; compared against currentRowMark
     private int[] stagedColumnMarks = new int[8];
+    // monotonically increasing mark; incremented per row to invalidate stagedColumnMarks without clearing
     private int currentRowMark = 1;
     private int rowFillColumnCount;
     private int inProgressColumnCount;
@@ -845,7 +853,6 @@ private long estimateBaseForCurrentSchema() {
                 estimate += 1;
             }
         }
-        estimate += SAFETY_MARGIN_BYTES;
         return estimate;
     }
 
@@ -1085,6 +1092,11 @@ public QwpTableBuffer currentTableBufferForTest() {
         return currentTableBuffer;
     }
 
+    @TestOnly
+    public long committedDatagramEstimateForTest() {
+        return committedDatagramEstimate;
+    }
+
     private void captureInProgressColumnPrefixState() {
         int columnCount = currentTableBuffer.getColumnCount();
         ensurePrefixColumnCapacity(columnCount);
@@ -1299,6 +1311,11 @@ private static int utf8Length(CharSequence s) {
         return len;
     }
 
+    /**
+     * Captures the state of a column buffer at the moment the in-progress row starts
+     * writing to it. The snapshot allows the sender to compute incremental datagram
+     * size estimates and to roll back the column to its pre-row state on error or cancel.
+     */
     private static final class InProgressColumnState {
         private int arrayDataOffsetBefore;
         private int arrayShapeOffsetBefore;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
index a525641..0abf8db 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
@@ -139,6 +139,14 @@ public void testBoundedSenderMixedNullablePaddingPreservesRowsAndPacketLimit() t
         });
     }
 
+    @Test
+    public void testEstimateMatchesActualEncodedSize() throws Exception {
+        assertMemoryLeak(() -> {
+            auditEstimateWithStableSchemaAndNullableValues();
+            auditEstimateAcrossSymbolDictionaryVarintBoundary();
+        });
+    }
+
     @Test
     public void testBoundedSenderNullableStringNullAcrossOverflowBoundaryPreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
@@ -1702,6 +1710,91 @@ private static DoubleArrayValue doubleArrayValue(int[] shape, double... values)
         return new DoubleArrayValue(dims, elems);
     }
 
+    private static void auditEstimateAcrossSymbolDictionaryVarintBoundary() throws Exception {
+        ArrayList rows = new ArrayList<>();
+        for (int i = 0; i < 160; i++) {
+            final int rowId = i;
+            rows.add(row(
+                    "sym_audit",
+                    sender -> sender.table("sym_audit")
+                            .longColumn("x", rowId)
+                            .symbol("sym", "sym-" + rowId)
+                            .atNow(),
+                    "x", (long) rowId,
+                    "sym", "sym-" + rowId
+            ));
+        }
+        assertEstimateAtLeastActual(rows);
+    }
+
+    private static void auditEstimateWithStableSchemaAndNullableValues() throws Exception {
+        ArrayList rows = new ArrayList<>();
+        for (int i = 0; i < 96; i++) {
+            final int rowId = i;
+            final String stringValue = (i & 1) == 0 ? "tokyo-" + i + "-" + repeat('x', (i % 31) + 1) : null;
+            final long[] longArray = i % 3 == 0 ? new long[]{i, i + 1L, i + 2L} : null;
+            final double[][] doubleArray = i % 5 == 0 ? new double[][]{{i + 0.5, i + 1.5}, {i + 2.5, i + 3.5}} : null;
+            final Decimal64 decimal64 = i % 7 == 0 ? Decimal64.fromLong(i * 100L + 7, 2) : null;
+            final Decimal128 decimal128 = i % 11 == 0 ? Decimal128.fromLong(i * 1000L + 11, 4) : null;
+            final Decimal256 decimal256 = i % 13 == 0 ? Decimal256.fromLong(i * 10000L + 13, 3) : null;
+
+            rows.add(row(
+                    "audit",
+                    sender -> {
+                        sender.table("audit")
+                                .longColumn("l", rowId)
+                                .doubleColumn("d", rowId + 0.25)
+                                .symbol("sym", "stable");
+                        if (stringValue != null) {
+                            sender.stringColumn("s", stringValue);
+                        }
+                        if (longArray != null) {
+                            sender.longArray("la", longArray);
+                        }
+                        if (doubleArray != null) {
+                            sender.doubleArray("da", doubleArray);
+                        }
+                        if (decimal64 != null) {
+                            sender.decimalColumn("d64", decimal64);
+                        }
+                        if (decimal128 != null) {
+                            sender.decimalColumn("d128", decimal128);
+                        }
+                        if (decimal256 != null) {
+                            sender.decimalColumn("d256", decimal256);
+                        }
+                        sender.at(rowId + 1L, ChronoUnit.MICROS);
+                    },
+                    "l", (long) rowId,
+                    "d", rowId + 0.25,
+                    "sym", "stable",
+                    "s", stringValue,
+                    "la", longArray == null ? null : longArrayValue(shape(longArray.length), longArray),
+                    "da", doubleArray == null ? null : doubleArrayValue(shape(doubleArray.length, doubleArray[0].length), flatten(doubleArray)),
+                    "d64", decimal64 == null ? null : decimal(i * 100L + 7, 2),
+                    "d128", decimal128 == null ? null : decimal(i * 1000L + 11, 4),
+                    "d256", decimal256 == null ? null : decimal(i * 10000L + 13, 3),
+                    "", rowId + 1L
+            ));
+        }
+        assertEstimateAtLeastActual(rows);
+    }
+
+    private static void assertEstimateAtLeastActual(List rows) throws Exception {
+        CapturingNetworkFacade nf = new CapturingNetworkFacade();
+        try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+            for (int i = 0; i < rows.size(); i++) {
+                rows.get(i).writer.accept(sender);
+                long estimate = sender.committedDatagramEstimateForTest();
+                long actual = fullPacketSize(rows.subList(0, i + 1));
+                Assert.assertTrue(
+                        "row " + i + " estimate underflow: estimate=" + estimate + ", actual=" + actual,
+                        estimate >= actual
+                );
+            }
+        }
+    }
+
     private static List expectedRows(List rows) {
         ArrayList expected = new ArrayList<>(rows.size());
         for (ScenarioRow row : rows) {

From 877ac06934676140b9934ec891b67e76df88968c Mon Sep 17 00:00:00 2001
From: GitHub Actions - Rebuild Native Libraries 
Date: Mon, 9 Mar 2026 08:46:48 +0000
Subject: [PATCH 170/230] Rebuild CXX libraries

---
 .../bin/darwin-aarch64/libquestdb.dylib       | Bin 38560 -> 38688 bytes
 .../client/bin/darwin-x86-64/libquestdb.dylib | Bin 44368 -> 44504 bytes
 .../client/bin/linux-aarch64/libquestdb.so    | Bin 208424 -> 208576 bytes
 .../client/bin/linux-x86-64/libquestdb.so     | Bin 62048 -> 62184 bytes
 .../client/bin/windows-x86-64/libquestdb.dll  | Bin 37888 -> 38912 bytes
 5 files changed, 0 insertions(+), 0 deletions(-)

diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib
index 836ff3edeb8ba21942ddce56cc1eb8d5e3360c81..38f76b86334312a2ad0988a896aacb6b5ef511db 100644
GIT binary patch
delta 3362
zcmZ{m3s98T6@btE@4v7FA3Us}$U`0RMU=PTE|1OnjA9g5qqga?EGw+SF6=Hu3%V=9
z6hTG2FjB1%sH4DuRk10|L`8!Z6HU=%Owz;_)JCjH27@5hvGn}x&7`xF-kE#u{mwb}
z-22~i|Ns7VtwP-y;ZT`R$wC+_?Gv1B$-=6miQdXYC;f^zTjjaOc0vZ2Q3xP`tW;14
zA%5(tkUAFxfG7Zmm*iIrP)0#gfj`#nQh*d}%PIhRSOWF)2t$(Q3u1Xb*xY+G!ouu))#V^e*0p>+_g
zc5Y1&*l=v55H7HJi?;ftx>$XjR+(g7%)H`4rcm?%GO|bxL6Q9wUP#-p2OO!d*vh!|
z*!j8I<$c+V~K1kLAS&kZN`yUO}4Jjrf_QhSO{jgA)UTNcj#NB+)phQ0;Tdl8Wgp0!NkFr222}NbU-ao~R9=xpUVCyMW0VQ<
zieTdZWFPiDz;UOa`Zt&RcvQSO9jYD+9v^#><1dn*iu$7S!xeDx6b7k;UH9D$+
zhhXmA>rnRY-J=}!gBO_b7jm4u-xE5XdBTgK0O;tW&ZieJKPPZeL(+nH-Zvu4A-N;i
z8}d5;GpgyOlbVgFyDY%dx`q>5UKK9GqrJ3g`0XW4BU*J6?m$&1wQKLOQ{e1qP-{XSahvmK=73IZH<+&HCzNH?`I%03tdt7Z%k)&3&M{&!3Z+NQ1
z2TdY=Pjt^mHC>~le!Xqy5*`sZe5KEN6|loe3&u;|rDS}=rK)VZl{AG=FGpZoUvfaw
z_z={sz}ks5hj=ehz`(4{n)NVs@Xa|ai
z?~?U;oy}&jz$1aeek0MC;alu=8_+sXfTHnTT*gWhx@^_y4Ny)swrr@xFHMgHtv9Uz
z;wpn7SCwrnFoZ)XP07!;8TC4=4XWvSQ^p(lTcMF^^tmZ(%u5W}x}DHUK`>>&OTh-`
zG0Z@w6lN=mk$jv9JzuC-{WlK?1?Ls;^lIV|B+*HVk~wGR}92C74LDDFLv@B&PvXh=&^ut(N23T
z=hfVv$r(5wQz5O`pR=0tem-F~=Z6ZX{}Ik!N+;jo
z+#TuUmz;CCe+=GXQvAj-#{D;P#ah0?Bb;w?zQK6_f0Ly0obzhVv+z7g0Y~@~ujhO+
z!RcSbxrK8p=Z6wY`M<{%{y!cncW1yK|Lo>-uI7Jw8qOy<7jfPc?ws&_&KJ1-2hQQ#
zz7Q_~X~A812Bi66DJqaL)6QaWVUr6xT$s7=F&A!d;d4LnvauQ4e^QL`6&LPy;ZI!n
zYgWH6*n7xDOIhE(V3jYvvj6*V0WLh#g~MDpnyJcz?P?e8Di>bo!dqN8-Gxmqyu*d1
z)Jf@*URyk8>{0nUvQ^k$&2E=loxL_yMVIJl%Fvkky~Fwr&T6BFj3U1|XR`y<>j|*0
zs~5Y~q=g7D4*%Li6%*Pq®drsHLZpU(6J89Nc+{OCKIjG>Rak9__zZ~X6fn@*(P
z_^L7Chjo71)`Sm^9{e-gRTGjH@nB`35LqN%TI1!`IcL{k$S-wi>rP)I+Y&F7-R-To
zQ54pYKXBzv<>8fGZ)f>0zqDMf`sXM1roq>jq#mza+S<@v|M~vG1l`3}U)}QEL05-|
zHItuPYX|T6Ut?z7^q{ZUtC}?RA8Xxb$G09l>rgfQY4_@o4Qn@7-pbuvpAnha)6nwG
p%o$0EDz-zjHY}p_TK`ABbALO{b|(Jp6jPkK=zu)8@w&8q{{_E392Nio

delta 2828
zcmZ9O4Nz3q6@bsX@9{T%V$AIHBog(Z$aCO;Tb{Cb<
zW>KQ=TB4XHtS8|q_>#OOl5n2x1(=Y=
znv~e|F`sf$`q8v356{SK_=cFLoXF!
zyf!`BAVcr896P+&Y8v{__BRK$Sgq}rHsXx-D7(kdm08aw82&R;v+8{Yy$do7pQk=4
z3^HdFsQQ|Ch+&MbQaMg$oX(7kdWHKH@&i&O*Ltd}&PD!{kOUK@puq&%R=2Pkay=MYmKlkRLGdnr{I7NW8Uamd
zZ^r-*Wj`U=aWQ)<)lqKDm&!?}c61U|PrKHnZ#S&!Q
zdARt(9SIDf#IXPuLI9Tg0J0dK$!TR@0&d8yP~ymm6S=FgH+MC=0eCLAklh9RB3H-G
z1E%F^*sFkB^6FN+3($L_AFkan!?Ayrz%gFoOMrjO%a+{uWnK~wUy-k2&4B0i8mR%R
z^GlSYG>KME#0&YcQX9ULZ;Wgr(_-tz!3nwM{jpPgh(lR}~jyWr-ffmX{k#JM4wm9Jy1=dqt0*ssEo5I3d$RS0dlnf%vz(4?7yHai^q
zU?`--;8JTnDvX4bs&0?V(E?*3plq?)RCkTp=5W()TvoQbYhX^Hr_3`Sa6HrQo;NPK1}iRx~1u06wGgE-Wo`_KV0yOg0G19
zNv|+W3In~eOB1FA=L((`>=e8p_+`OUfq2t68jEV#mg2x573Z55yT5u{qL;U$qJ}`(6gS&zag4cuv5|j$A5WFC`
zU2v7)3Bi+s7X@3yGmj0YYm(k#UOvwc`u{O-E(qevAg&E!YY;nw*c-&pe9uey;pq1U
zo*(*x_@y8o4&qmX_|HLnEr|c|9n;qEHiIokl2UGeXAW4u4q}EwN0Lm@bO$`xJ{}hz
z#7RNCHi-2Y(Ula%Kgu#lv?Mw_(PgI3orrGtSbO(HW$ZvtgF^pl9^CzWc4+T(GJ{C^
zeoM=bN~P$fX0z5cHPc0DsoQ92App&m{=`G^KfCqA=$GH{UsBFx?d{w2x%y1qRad=X
z*XJ9y<1hP)i%;jJ{KfKk;f~){UE2GHpQd?-3W^^Z*laly``LzP3)(yPeygk~*TdoL
z8x>QR(~aS#hNlZwk6a${fAZ|Ah9|QU@@SFWC2x0CD2?mypVhg;`i{@COVtZ8_v_gK
y+q==>&y8;F?SAcblWXz*+QHYuiY7n2s`gYP}L@O&SCjsFWJ%~m%6

diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib
index bf0a00ff9a815604f95c2cf8b437c264e52dda58..72d228a183138372ad2b9d9eb367f016dea1ed5f 100644
GIT binary patch
delta 10890
zcmZA730zd=`Umjm9A*#{1r!iOWD%FEh=Ti)G6*;%C|=au5doE4AQ)UK9lD4%5{VwS
zuGb>}-fXed=9;F7Ls;O=1XE?~gRjN8We7&-}8hHhN?azNZ
z7ZV*it
z0itnc+YhGksfo7J2ZMMIo$dPZ`xZMa35i9IL-&VpDA9BH|ID^aiMB7DfArN>wT@8#
z-B&x%I@IeQeZ?+bG8Y|kn>_WprLT6ob&eX}S4(SCtxoHsb@h3v^ILq#dBr%}wIti)
zl5wZZCCN2$CGlQKrOA7p72Ig6a=z4CJM7b0HTBkh_K8qC_11!Y+j^L}t>{5Rfw!|h
zT8s6aqIT`AmHURO#@7`9>J0|oI
zM|p=$MUO?!f+somtihJH&a7TqihmdFPTRqui~o}OYfrhgbIHXr^GZ+7j0^aZz*SUTQcCeP$0(&`z!jGOtqDJkIZL!OgsH?Hfs
z_Tu^h*ZyW(bYjsLA?6~7YA*Ub#5}OFpw@KnkYe#oEJ-GFWt}0>=Hbe(-4bm{u5O;s
zIXu1N-{rr(=IZzwv30X;NqKeryK?^vcE;J3?ByOPpE+}EWaAFlF7Z+4L;4F}Z=Rg!
zlUU+Jzs$YjT*BZJUjI}k&%{G~Xa`I-O)*V1O*5rUtrBmfFWUZ~JwAhCxNGh$a
zf8PH|MkHS`pJasew|~^t-^X`(N6}?xO%u|lO`15lq0U@le5s>$en?>CG;XOf+aB;$
zw!nk0#G}~GYW`~S>sY<5#277V-qArF?cvXib!q>7FwvxGX$|6<@buEqX-0^)HYPz0
z3(+pb>{2gv&{mlis`ol*Zn1Bv<2q>X#nvPh@rAv~t}GHEet*BH2Io5N#1qZ}<6fc|
zvKgmx)6=_`%yu4@yJrfr9~fSs1fAr&o~dY{VtMv?-tx7mdwkL@;IBqfY0Eu<_R;aH
z73&f1n}$xu^s-hj&*|j_z5JhE*6HOHz5HG;uj-{!FMrg_n|gU$FYER47rnfrm-qDY
zfnGk;%SU?o*s522cJWTAdTG#0SG{!8%jSCN!DX#CQKw9KPPgadE6zT?yz7go#R<)e
zqVT^^QWftO2Ia&Aqms+(jpYXM7hx7D@->lmdW4rTfO7ev;Q|es23WMyIQ`&m2nI2-Lu3|%86vh`eQEx)+!e^
zwK9?X_%1Et3sIj%9fkD<_L1$IP(MPw2JKV%2|}!2fjXGKN<>yKD%4+i(Ix(#iG<1g
zEfn<`)Q8ZX%HLEGKaP6jAn9PNR}BvzEaL}Jzliq1Lu7mp>fX<=t-Qe<;su3>LSNRH
zL=~dH7DaumD7{D-E^9B;e;|Gvbr9k$Q9nfd2I_W*H%EP+9T7w%rUC~0({
zIlpwF`jsq&jLo5P&17kBtuTzW%%v)hI{IycjLn`Wwvm;WH$R8o*G3n#om}lYFK0m(9dQ*$>;&i6cM>}k
zF1FZ-yCOCxXAa$T9iEq!ZposbUE{LyEqOVM>7FYY60+xJ<)>xL&B^Ca{;KQf{50C(
zIzB7Ik~S}E9-Y=I3j&qjwTlH|t;IeA&pefz~`TN3jV_@Hu+e8@?aC89^KiL=BJB?>DSg-f{D$!C)q
z`y5?!OQ4%>q(;*fx4u!)^s`v&6|rg4_zCpLEj2Yh+DfmwyU@{iUac-cuD01dfwpk)
zehLR`M{zJdnm%?H$Q!D_N@6?nSUONlWw!JEOFWSx#^aS*>u)Tiz)YQcF|27{axDGg@e>u
z)!us10tMeNQwyS2{AS|rB9@8l4(zgE>6hVhcr&~cu7n-%8Ms%dT>m#X1`gooRk8gP
zD*{6}Aa-C5oB&S=lN(5akHS;n=^bVJ9JmZ#2v@*M;1h5WY`ubj4S~DxD%dStzQH=U
z1-uFF4!;ejz}w&q_(RwV+oi3v5rI#Uunj&4?}fjB55Zr-HSh_z7Cr;l!xv#M{_cw9
zJKa{E5j@(KoXC%6yX6&?he;bHI$IF4-Dtrc>0iTCw
z!FBN5mV5&jJ3w9JCwu`3A@BmYEBq3i0>26`ggN7WLqI-i6J^3RSYaOAuDd*>
z3t;)kPxKPp1?|gV4K9V{(12(STrS%A{b>^d9ue{lw!(4@!Gmi!0PP>c#qfSujxqRp
z0Sz
z1!qUf_ouc9$k7i`dw4;l+@VhJ6*v->qa>a!hJQl)5O{J=xqdt>M^Qx2!n1qopPwid
zfm2vv3M_|PMCtHlv_B6gW9XR=%V8POB6t$oUxB}bSHiVc1XdyN4ZIQln?C#^dK;F*
zHKKRmerVqTuY~u)a`;Ac0N#N1FJbEq1dbyh$B{&*;0I{G2q)>|N}{W<9B&fcfG48;
z4*Vhf2Q0^>{Lts6pCeY2d+c`a3I>l;2gL+EC;weDFDBW_JMFM
z9IKoAUyg){Mj+uL5>ntOeI(4!gs>bb6HSAMqJ0*;2F``$h?=MXE=T)PVR8SzgTM+T
z$e}mUYp|jZxrx@n@BdXus6s+peaKF91eRlc
zo_d1A(S8nI1b+w1DFL2~g5?wd(Ji8aLu$-$PiuUII7gwt9kuU@aDf;9EQ35O{HTb6`cskn0!AIaU
zSWbTMy#&61_W7_`p9JCSgcSifJ3_Pso`8fE@F93LEN4-O*2AaJ&Oc=7A1FqjO(A+8
zma{EHJKz+wTR%nMGX$z&IUPe(10O;ANqB%hJwtQ>meV#wmtiy7Z@~Y7@4|9=hx`8_
z0*8=bXe~cMFMXbeU&C_#ho}u4gZ3bJJKP19^F%z^2Y-h4esEWP+KBr<1_6CSiSHNj
z@Blr5Xf(VPo(Ri%CmyoFZFrhNEV*!JxELM|uY{i!7Wbc32&5rlEnEn1g?A1UnPp;;Sw4xEQQV8ulG;th9%{or^w5S|K$!1Lfv@KTfh`H8wAunY-3;Z<-S
zSk5i-bpieY?IYpq@MO4LpZX(u9zFyYz^AMTyokU#*b2+JL;h6=mUE3nD_}V}NmK&M
z$xJ?{U^)58liaYJljK1vY!$PZV%da%ob@Dn3zoB_d~<>2EGp49SWb2leE`eZQNGZ?
zayFGuT3F7$677+;lAJXq+K&V|%}R6-meaEQLkHY>xZL3yI2S$&FNIISE8#P6IebAk
z_x~;gE+gRpd=;*Ne}V77_u-H@xkHcO?y%w`PvXI_D?A-;AuR6y^AYeu!csU4u7JD1
zC*UFQE!YJA3MasB@p1=8!l`fyTfhHLL|_~e(%?yO7MuppfivM8I2-;qJRklL&WCry
zi+uR|FK)jF5LkkQ+i(%wEJ1#P6|f&%0tdlma07zu=`k|B3cGlTQ7=EIpmy
z8|}Hs7tbwE=Zl^Z_Tq2ZD6xqY1Qz40OoJPrU@UOsg~uk$UDnt|YtcSE)V`(cb;ark
z;s7smjPi|g`3Q++GaQ5VGmY(V3s;#R6Dacs!aLzPa4sJ(vAhA-Vf)q6R;m|=h?l!a
z@Cughz&kBIc?dijPJzWw(W1Q&-U^E!kHvBnE`!By+9H1^EV|iK{H9RP4>V$#pa=Ne
zhS$Qi@CkS%y80S?2bb_iaEAoBJ#TmUhLzlPVhM+5Vf`p!eNNe5CX7HrFW+NZ#UW{&qWuI3jg#C5;l>{-zJeQ{j84Oi
zPf}Om#wV$}aN}KBX|DfY0P&sCm(b??o#OZZGtjL`!jL8&-Ner|@w6tM*Tj}4Uedhh
zU-xKPlZ2H`yit2&U9k7ACeef1x9fsKk2i^)Y2r&-!1`coeUs>KP2AAL9xeX5hpn49
zu!%c1anHZH9S!_zK>y#DA9E+zwQtvF7?1L+W7^^M`SO*0;D(vT7V|U5=HQoPZP&&C
Wt=A?G`{sYWY7{?x+k>}6w)lU`uA!a)

delta 10434
zcmZYF30zcF`v>snUS`-7K@>&V6*oi#L^IrAK!gNC!6MBU2u3B@fW
zwZ2|08o!|-$@_c~>C<*u(fOneKTP$mogryJw04FTt4#G?hp-dE>t<+cl&L=T2p>jR
zSk-15{j?6HRB>nps-NQDQlk!4lyvPCr;HR|m1rDMl^4(8TSj!)$TNt~{8zi2=(s`@
z*GS|wfT$P8()mwJ3ppbOTnC7i7_*)vx?Q`d^z-%OxLC%kAH4t2Z?DZCggPoo5=w31Y>yE%C-DdRrTN6RqlSuZT;oGv4$dO12r-%_eeKW!xg-
zdHk8f9LZL+=o;
zXpw5Wki6=E>-+`ZhsJ0>dzYwJ$7$L94yk>{X(RhTH2ltJp5Jyb(RL}-*12}>=gDi+
z4kxai?v+}Xw#!k>jkZ?D=xFUo|KaMDDD6uBaP^BQEy!nJ?~W+(6?Ikgbu>k3CZBuN
zlTliePl)Jp)y^tRGuXH)AQKDC!mGD@{|@h+Xng=u^I!_pLP@eHcsO#NibNZ%)}yS<{|-CEAAHTFmRY^}CK`WN)R_?CEq
zKl4SyU!9xnj6A_w`RIN!N{bE{t9piMc>&?-#ZlUZfRXCXQChE{_xi7i=lpAxPPX<|
zaZYC(GFsxb(?QYWmPAp@T~6Q>X305H~qCS*Y+!()tB_RN4z#Ac+jL@
z<2aWC`gUF4PvlCT)jRc!FS&{DAY?my{ls4J;T58K@~qMH*4=U1hTzDc)tuzQfxV*G
zC5osco?EZA23ryj8SjtN+Ja-%=s3+kBuV{anwB5($AHeMMDtrY@vo_C*n@GJWr(K!
zG_A#LsE?wap4t)|>Zhu^C$~%){)FN`bQ00BKFQa=O06?`#uCMMh1#W7#)=~L
zMQ^@sY+*YN@rlHr8}zod#z#d>UBlLdwpgZSI{C+h5Y78ncGtzfc0zvW8unI*_Fa6E
zS{9;(n|7*WgS88$WolBemX+|F`eKlFBjIrBslflYDThUf*FPYt%dt@*`pi*b+{Jfm
zvKe=9)2*Agne8|tH*Z5=i^uc|C142O*Wy>*u2}B4pLg<7)HOcqmhu&nTIb;s(DGc;
z2F2>{M!fxjdKs*jVR{*^ml1jysh81u8KajI^m3wJ#_Hu{y_~9-ae8Ue%S63Q(#vGM
zyhATj^>UU~uhR8$j$UT!wda(mg%39HU!9PO!CkxLB4Kog5|WnS4zPA3$v%DLrMhtlvdFR^N0l
z^2L1H;SjzBius>GU5fU8d^Z*Gb*PPe4`y4bdjnpGlnDlYAc#aW>MEH?2jLO?h2<;}c}vi!U>gxEi$u?M@SA+=}`p
z>QB+Wj4xF&-*2c(_<|M=!*YLw7x6_e;{PLRD}TtIECRuz&?i%zsDeeIPfhWn@}ixn
z?WlL4-h%o#>JL%Bih2|3qp06U{W8~lG`#331P-Bo5A`zjs5OCh~&a3cEzZffRZX_#p?m&O+sEHy{ctA@0SLQ6qjZn=ea8#0RX
zO%)62xFIvIbXG~CxiEJ*oivbIoKH;#3w_B;X%FY6w2kJXoe!_diS(3lZhk3kHj3wCDg1b6=UM|8F?eb_MH$#6(4Lhtst+gq?|g9vvP~MUggYN
zDB^h)Wo71KOJ+flScbW*tXS+%amB#-yPS(kmYV59XYqx6#QE*R#0G_m#SY_d+Eh}q
zgw8o9l$moaX8P7S(OhmRD_KsLoXL<>P-rgC&MPb_=RLn+yt6!;nw;-4=UK9g%tdra
zYpd{6E^B{Q46z)rl
z&52%~#lfmn4szC3w~XoI=w%Mu6FJx@Bb0>uz&tubX~$cDk0%
zSWZXW4D>XgpaweesMsab-Sj;#{ud7a{-~me8L#r^j%v{;Hmb^9->7x&;*>t;CJs-7
zmcJ}L@t{ivZF8MloLgA1sMwtU*EiAXy2xxvEGRF{wd5_PBd&{ci}MT3(+i3%Whuo8
zCB?;jVCWN9acG{`I+n$@#8fU-tj^r8>BH+34-qj1vIE(1{LmCrJe&^S2j{?#!KLuC
za9fa^e=pn)pNA<}Z?}@tTW)}#3{Wq)cZgg-U-)4-1oj#t+auvPcrsiJo8XmjGTfls
zN;C_B7m<(&?}F#Sd*B@SEW8k=p>l%?;eK!#JQl8mlZAQx^auiXBcU2z46lJLa6McJ
zKM7aC&%#acR@e@2hYz#G`sob>+L7=!+@Ig!#PmKq47S4);DhiC_}_3Ad=k!w&%mXj
za{Y7>0r4U!rc3Zn_$T-y_&4|q_y&9i?t(jD1Ha6OliZu{Xkv1M!-nbm&y5HqA)y~U
zANGashXde;;34p8cm%u_4u|XED7e{*zyt)uJy%SV;4^R>O!`|DQ5qZpXTb6BJ@6bj
z8#cpc*t!^jB?xSUAAsds6wy-nJG4IpUx%w;=Mi!TYT$wJ8h9xDxU`iA`e);IrG^L`=i|t
zUJQq_tzrT4RhmbnkWh()X!s~R36`(){EUFVM0*OHpugf1&4J~sKhfQAI@)vk@bxb?
z-~$8-kRbaAM8)tCv@e6lkCrD{6)gJ@+|`FoXkQP%0XM?3uR-)|A3pwK1NI={MI?*}
zmn(P!mYou!7B~j&AHYrU$FS_Q5FLegp#2m)XpCI`H&z5>?}q3iJRAu>zzy(k@H21+
z{2Z)!${p$zAy?oI%U%+Xxxl_?xB4PbjX(%2yHiBN;SFdX3;!3M2+M94(G=KytlWSk
zcqx1*EPG#k{4)`#LBay~D|jI+yJJL4;QvLt1&-CbV?+Ti3|rwp;n!f}IKhY8R6xvV1L3*E`=qxP90EpV)XtaM1SHr);ax8*}JYnk-NKpF9lY;cY
z3I4eQmV*}jkb%9>J{VpBhr)7jgP$evI<$|2JK!nOR+6J1L?$G->Z2b#f(%>WyI?s=
z!Xu(^HQLQ^J6r8<6nqCEfVB#3sD`s0_}~k1AZQs!!krK!5wIC
zhV%8|8Geg~<({ag56ecXrWGAze|h_1keXuk=c
zfsNjH{_6uqd>?Sp24ydM4<*5E(kr@Z<2iwy{zCReZ-2@~M0@Fe(U
zI0=3oPJwsA^Wb;kCGZ}&7`9sxcmRQa!)xKs;GOVk_!N8=z6!U&9&vJq{sRZVKfn>N
z^%n$U5cnOQ2H${9aIb#y8>@i5;HTkWI6a>9X)J7kW8rFPD@{W{{8S^RL|Bdu@*qDf
z#}fIz0L#Hg9>Ip?U?ou|EC)A((
zIhe_B&#)Z*T7L?Gc3Tnt}^SHenvc@l1ho#7YYUhpp13wBDB8{h-GCF=W6G#CL-Bm}`ea475t
zkAj2X2zV|$4qgCHgw602xX6lt34tweGW;H#3Lk{i;p6aJn5N4;Tmbim^I%`N0JaWB
zpa_8g_yKq*TmhHE55X&8YY(r2A47XB{0v+t%|Fv1(13*Z;6`{4{1SW+ehu!IB=@)(
z9tgh!hrsW{VemfPeEdfv@Cg#`hEKuS@EQ0~_#E5_e+xeaUxq)0ufW|uw_Ovqy71QL
z$UW%i!vVGz90&))F>n(+v%5W4&Y#`g4j04QVJqAWuV-7u3hf9yjfCX;F{ni
z3*O(uAH!8>Kg|ww@<_mk}|&N9Clz8x+dAlnzgZCL(BxLxcxPp#5cvSS68k#GVI
z;G-$Fp91xKN`oWeGFbflE2d3wJ$wKb@2+CH47c-JfS88!!SYo2qRBzvK0Uy=
zCSLL0fw$osKMH?>`_={c0r*#V;1s#SF4*oX?c>H9AYPigUjaizfZhGd7zKB~GKyc^
zMSJ%PmiTpDxcldXd*SY%6PCc;!;;J3?t#~8wpA>E2e0+14hh}AKRgY0zwm5>yI+vr
zhPz*oTH)@8?Wb&Bytowf>6aevaN~>S)&zI=f6frl!^3(wyoV?Cu&IY<_V66{l7Fr+
zt4G4Z9xm@;YY*4;@WvkAtgYM_6e523y>++W?cu%JPR=^sBl>j@w`o^4295f?N3^qt
o#sBHs$$xKY0ga24X02~ydCR)SJY!49=EsaJ&09zIa^hq2f7Bl2W&i*H

diff --git a/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so b/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so
index 4609bfdbcfceef071e6e7492405f07927c0f82f2..76158e167fee131403af79b61a477f0b5714af2a 100644
GIT binary patch
delta 4853
zcmZ{n3vg7`8OP7#?lYTYceA_ML`XIP@&c0ZD5Mz7CeXOFFvP&LK{^@)#>jwzGRjnH
z7DXmBJ`yhS;R6WDI7x=qAhZ#SoyEa6VB0}nA|hQLcB0ab2uOI|{_pPH6n2J_Ir*LM
zf6n>Ncg{Wc+&!Hq9g(jbZACoNUOdrWK9t2$`Ox-7rS`BdZiRjAZYGg@#dJUYtO7ZuGf!1E>IVWDzDvI
z%Z|DjO(WRp5x8N!|F14x}(_|RCJK9ir%RZp8T)$bL*>NL0UExKiTH_=W#X!i4{uC24Q@$6XB
zHcNT_16WZvOJM}KX!eQGJ?vFWteYpL8752e
zR+19$9yRn)Mt9X{AM~oiR$$+*M~h%shC0^v%ldp{(6BLRBDPu-V$Ixl#{7bz2a@y;
z4L#^qPuT9W|Hl}#!>i`oA9u|+W*T-G-qcmL%;t@8gQXgMqlsE0@(YIEXpH4B^rMEi
zNK*U7#?&H4`*NfGcg7rpcebC!3Jgb$cUG{+j1C>fqIe8PrN#hZV}QgGKWZ#}z^6W$
zvc`VGXcqPC6pp9O)R-epo`sJunEk}8`SX@9U-%?@3f7XR9;MS+a?;Lq?pBkU@1LNS
z`;`>V1pYYwggVdv*ie0mbn7Dw;WWaa_6-P7u%kP_@&8$7%`#*iR<6V>0gTwWW5}0h
zcJVslY~!rWwI+DgnXj}5MD8(a6num&sI;#_MP86z5xnp=^Dlx|@fx}X6)2Frmj98K
zNS@0-pqF*L*ZX_P$E8ppl`G9RC2PMtS9JMUxPA*`%VyZv>>iM&?m0I!x-40D#Kv~bGqVSxGPAqN&z9}Cu-RKh=BIf9_E}Yc
zU6=-Y!E=1pY3c1gE4-g>^C_*l0eMu<0VbPU|1EiUej{to3$yl|mx9wzDg4H%)oeyp
zIOoR0;1J8NjCmeM@SFvAS2>&6vOLaapGNdi=_wId
z1w!RT%XclbThn#(*}l$*eE8Tl%=&Eo
zxYic~tUWP9p#;00Zwd@-MP@K*JGFL)}7)JxOK|b5i
z2s7yJ`0vWY+fmk^RrtAaZ+2bp-R$DoV)U_V8a3L3SC4>EnbCbQC3yAbvmbXJy!QXm
z^(dlv@G10l1|KO-UL&`M-zSb=qk0dYkaYrGZ$7)XlVK_K6-WfVnznj)Mz*StATT$>
zv)*I_ynk}7_n#iV?wj?ZvRVHo>0gPE&@_i&Yy}tqcY)(UtJJgxun1fVhQQTe6Sxs<
z0ee8JqG`wVdhmvBn>4KgU%yUp0a#?#w3omZut(RyeUde^eiY8>6&6kN;Jpi3@pwS1
zP1DwcP2dN*ouX-@uy;DaMz9|YgGCNa>j0a;2-pH10sFuKupe~bg>OkkKwv-kYp^Iy
z(^i_B(Gi8WAasIzz>rhZoH*K=z~NvESOHpHn$`psfh)lf*akL%+rSp^1F#ePJJ<(C
z!G7=(Xmx9v6=!l0I0OuV<;^Ixpil?)flI+o4`!s>;4ZKq>;tV{O*;vOK!!uI3G{=V
zU%$ADwN;dv4S8bAKChAAN@A{2+ZJL?JICmxPb0^b;xW<$bhFijlmM
zHc8a2r%zDlJ&3)Ra6AMK4q}gYuj>Ys3B^K8OMLfli
z&_a_K$Ak2`2|XIkbV~{!9W)6m57QUWydBtcSjbok;C~eRZ#$)$;iHenm_-JB{LGBF
zLbME`SZ$#~mI~=G-JU^J%BPO4Gg9A@`yF
z9O`4=>*};H-{jn!Uh|q`L(2W@Q`NL78*OA4)kh9jAOC#HsjrS6OJjN?HEn&W-iGWO
zZF+~gBZq5_f3Ew~S7rz{b~#=QOrk*7Vq!VON{ET?tsc-+Xd!4zDNhl8nrsp2&C8)}
zjCUUF+X8J{lJ*v~o+PaY+MXnBAGE$C?Krd}N!od6QJXe%aOq>u7;oJ9q=p`714&vD
zG{ufrITkWj0IY=Ov{SZKWSjL@y4+5a^zurS>+JNPRg~0hN15RWNIY(R%m_3`l6C}|
z|28dvcKy(XJLs=gk(1-V8yvi?9@ViJ^b;cFh{rG$<)(Of(gKwAqsXr}GflxrEN*}&Lms^Ii#mpG~nOJb_=h{QXiGv5)bhXS__j$sq2;~|nIg-$pSCLU8NEfeN!CgXM6IcE1l3s(eVa+HKtr2ZzM3^v
z;9GWEs*06;%t<@U@)?|j#TKBycdJFt6cn+cdxKp%>3f^3;7h64F3TvCB11ThE~Lmz
z`aVVa_1ICyOk#+{5sjgl89$B_W5)L_Za;U?gLlyz@1lqAqDSI-eqvk}pJ_S^S?*kc
z6SPB-xtvj+19OOyD^+HWP28puy$7Qr<&b6B;iP7Xu`6tJ0L|uf
zB-JO}NAIEsjE$2RHsP<%cIRAvpdA(&@ZY6%53Nl_a8CQ3op8y(iBKX^lqzvAPK
znVvKBMmK$oej@6jA9RMjl<(EY^U)$N?1-D*(8;`Xz>9#Hn`(UWyn2Yc$NUy2ls@T*
zIqp05#en}m7~V=CJXDM*$HB&M0vm1PxeKYAAw(wvkJL(wZhf
zXl+u6Tttb@B7P*Hsh!etpj0&v$)2UkI6?$*Lwt55Y
z=PLq9Xh*s#y*4TG-DR^?{UnvmZ;AOu
zLW>6(FJRvY%Y;%(kmJau<|*;iw^3j-)DO~^Y_M72T;!)6yBbU&*C0QKT#H_DsH1y~`Ysjb@c6#x--PXELuhuX_s3Hc!&L<&Bz}
zceD3%jj;mrLmuK4=5}u(zfA36S!niqqkO!15kf4}#PGH8mooNxsFD
zRXpXxMU?&uvMvp7(qH@VG&=EA^>^;f)hv~=F%eH)?1uAX8%t}k@-mMnd{
zqjwnQjlewpr1MY~Hs|UanoFILrk$a=Gw_k4C$ggTIMMnXJ=Yn_LB~Iq&SZW=gwUBW$9Ip&K%v@4D1W1Luu#1?&MbF*wfVFOnTIL@J?r1lbLBX
z-_KdAcRGDcgicJ)ec5MrMe9tZ|AIST#WVzK6#k=zToruczOU8
zD3QFDFQ)a9kL63~rwU*0{Q
z7VuKxzeoEEc&(UpkNyc3|KYAh|C8)!$A7l{J$%oiFHmmz(Tul)_qykWc(Ev-2ZyAd
zxLpmVW$5UCa}u}jzxqLH&Rwf;Y)qbYb6SehCfXU{W#ZYpbUec6iNsyH5#h6n7NF_<
zS5K#KBAn`zt3Gy{T&0?kJXC!54xS*`DL1lmd2q)nEWg{mVI<#sH~{v8*T5mrgXPmY0v!T}z^{VwJj>eTYeqv9_CZL2JzzgLpd7(8*qCow
z1~y(RI1x;NbHILZ5jX@ULCruwFb=kYb)ed?jbInp3Z4N|;5l#zyw!|CJcudB2Gc^8
zRR_kw6<{5>5o`o^fGIEq_Je((7DflbI*?&^HG(m49XJzg0~^47;5twxcen7Z^~|$*
z45x$;xshsyxv?EG9%_3x&}tzGOe)ScNmmevF-19#02lnNA@w;d=s5j=%U*m(Zny%daszl>!{N!
z#_~owtH-J~1IOh?VIMDvpjvF##87u@C)RLR;X$chk>&X!aic-X{tJ
zG5Dn5GeDohR3GomWDYo-{
zWNC|_HD+l^Xq&RMP0(7hv{q=lv$TWI4rta3k3F~x;-M_ZGtj!Sv~$pUp}AkHQ5erv
zX#E;34~Sx4uG&-@z2g%jbv59ru(bdk*F@=p6)5-Uj}KT~2+m|_JD?2?(_)Bo0NMqe
z#stKeF}*0?8di^Lruy*_be3cH<0i^6ryR!qOXSd7=wI>7O!?6|kD*B&w5lBX3u2F2
zmnlE0S1+;!S_~%fpGiSV-50T%B)(xP*&k)bzLRBd&9c=uPo+E?Y!3w<
zh_PZYOr;ORO1_bP0+S!6BM(F+kJ2Zw3Jw(6&ge%bU5k=#i3%|orMZ?E!=v5HEb$vo
z2A5N@=bqtml|JZvtS;-AL;*qbY
z%JE)VE6~xkUimB327U4kUQL}o1lH)3PrkxCXqq2LcOUV~7X%r)oP%xsye=#E>(Zcf
zKx)nIea7vfy{~du%(>@e>m5(f`<|eupP)1QzC@(UAdgYIj&3JxBr~ARx~${rOnHiX
zeRF#=KU2tO*L?V(4ol~$#xKXxIbB8(Etrd-NgFL0h>j3_KUY@ZnCi@xmBnf2s_cev
zdx2Gs{)cjVKb
zf{2?4ci#!hSzL4&G(7}?g=tL)j(z#`K?sYuB}95yt}JHGT5-?*3GV+JgI){Eis|lJ
za?k$b5C6c}=g3TFtd||V&!9_aClVnlh^PUEXhB3h4?9cQu^|0cF&U!IBIt0^pfw}q
zReGsV7Vx*b+Y9CPJoiLB@Q<1&fQ;Oe-X&l{*SE-QoXE_ZpDE-E?d
zT=`udr*Ttcr9SA$IEI!_k+bw`j*a8#E!aW#*m8-pVv20$RkXTP&L6R{;L-bJ=4yY4
z`bwoS{75=qDqrVUX>}R85Eu3Wa=EX&vkbAY-^P{8%7D8-G+I`U;b`<`xttv^oMkhI
nK7kr(5O~chMQjuUyQsWEmgxnWVTLrWqNmuq1vO5&N82o0=S)j$9T+tYR+s$)^pWih9wqK9G
z{=!iYQ?zz5eq->XdKrMB0wfJc5+@j>nF$3er4$`6qYh@`hS)_|}UW(s1{4T?9JbvH8ZvuW-;3vD(MEGf&
zY&hB!xYO~wlEYWQ&BD)yUmkul@yo|=7JjtN#&3@BY1&+N=fN#z|8;P$$8Y>sFE^03&Gf&PKal&%>;E&LY|F4m9=P-Ump@9nV`a{XM=$@a%X?d#XTWbawlBTt
zyY1VaoH1?Uup9n2Y2)hAbt^rP-L^{2FM^!yGH-N}!Rpnn|3eRuM1bh_^Nc@gvvL5J>6{+S5!84>i4j8LxE
zF>t!me_;gunGy8N2fwhJdRvA`pgaASMBtA`;EzYh&z1=F^S{VTcXr1{X!oB)(0?p~
z{P76+d@w@0FOQ(-!wCK7V1)etF+%$~BG|n@0>2_cep-dy1Z@jXRxX5TTO{(Cpe55J
zg>9_iXg3X{dQQ+HR1eS(ccoZbe{C?VfjCc`b^OFUu}Rw<_B1R
zT9yIia+rePU@cMe)UzNvSBi2aXiZZMz>c<;c{+7qY)cJCD--RWpk+-mz>k@i!lnE-
za6MR<`)+4%W&a<~@_rr{
zGg*Ex_lGUqzLztn{0^ogKUpB=jhxR`&gXT^|H<6Nda{^ncKfnwXN}$MDXsO`?V7!^
zrqZL?E8uAM{E|X@xue#xtkUgq)Rq+HR5@!LC8bNN97bICG4`@mrKF&=s`566HmkJ0
z)L!Yd-&*Hzd&-yE%c?3JH6GY2w=Z;*dF<7W>N3~r^uEYdJL~(T&+YJNwT{xNDrZ?w
zYL~9|INbK!%35S28~)LFFQs&agAgZ;Ztj1HNl~p<25$i-vUs_&Xo354h*c|3GRNuw0W)tc9
zP^w#;l{Jwn*{s;@s44G@9Qs3jU*ud4$BMqll{srFDwoyOI_5ZQveA|+Dr=T$%S&s@
zs~mZiHJ;l1njB|M4K-{Z3JYa@6-8?o*OX`1RrEzK`c&VAp&5N*jx*O$RT{B>o2wq$
z^Hw_Rs>+vA`TLR|E+=(@{F(}9Uy@MAk@D#pg{bRQ_AxcG*p=8_<3d7D1r4>Yqr?~`
z3(8%6$uDZLu&&BeSyt-yL${&6U*g1cQB&^eQ)L{j*MwW~H4~%bmy3=71?2yFIbK;?
z>2b*9BG6mvYpf;6cb)Q6728
zJ2UZ|(i)_N$+8txTcjaUV{l?|$fp@c#lop}X;qa=YdsU(&IwlXLa=Q7cnVITV0Fk>
z7V`1IHp1G>g8b|pd&Y!}36n#9>!gr(;)INle_F^#CnwS3Pr61ZT3nYVKk-=n;)Ilp
zHHZ3X{Sg=SU;WdBwO`@fj?RzL@T(h9iIb#$+F>x-qLAZ#98cl_cQNMcn5RsAtRmaKl5(pu)?n|HF&bZ
zpJbk@@R3Um`80+9k@b&N_#G^tuJE^+XDa;KGDCls!kd}v3J;bW@@v{94ZEMuoq}@=XdaTW;vttnhy@Z&vt^S^vWd
zUtDSE*`n~*m~T^f@hyhDU*Rt@->>jL))?}w3Qu+!JfQFkn71oDoq31CM={raVd#_n
zIf;3^!mnnYtnhC!PgVE`=4lE~U_Msi)0w9${4(a53LnlqOX2a%b%jr5UZn6#nJ-fK
zQ07Y%9>;vS!XLbq+v{hBOV=@R=`pkgOdJbhXlpm|0GEBS7I$rF
z&4!59yKDyoqdVroE9Lb3HdT
z3oOcHzJIvc#D^FnV&81y;sFDWd)UN>F(YS-iOYL3#J8FFa00yfn+*@;UtaSjvbUM`
zM*hz=ykft<*$9yQ2*WG(2b+xm@$(F?*ta$t0pcSKuh<8gjR5gchF9#{n~ebR^9`@q
zcQhLT;`0r!vFG)a_-GT4H}NzRPd4!jOgz=ZFEsHq6TisB&w9_+z}Xr&TLWin;A{Is
zbP>Lj!gMOzmL|ftQTQARCyQ_`h3S;GO%vf;C~To{`&R%LFQqVDLTGCh;TtGSr?hSR
zMR-1i=~TAOFT%4ZOsBAITSPdU!gT7|)-1x)DNLuVZA~JaL18*oZEFzW%P35zsBJD0
zzL3InYTC9$ghx=APD$H}M0hZT=~T2WON0kfm`*|4(nUC$!gT7{mL|fTxB|AEPC46>
zMfhV1)2U{gCc^Jim`)|z+P|doAE9s>gBvH^!X@
zzwV1GsnxW=T;^AUPc-->;!~NA1Wz#dP;k)=fgVO~k39*`Gvqk}&u;RxP=C}LuRe@R
zLSTo;nbzSMhVGnH4L`c`fMBa-bOZ9H?PM47j|ut7kB~F&E8W+ww|sc5-qI1HNA1=R
zeB~KR5Fc!_O5bqxtw^WUU9mv-UcD7kDBM3iiBNktt}q7viYf>!M>Xw^
z%ZFUlP13G(+ipi>#RSXfb(AZuN%y^~`(7;Y?Fe3MMFNCh(|!LZ!sl{0*SA~uJr}$<
zio&*%XQ-`;b>GXPs*isW4CecGQDLvry`QITPeZJ3$v&j}{t-0^x$J?HwsX-stp^rPQ
z`<~SsV+Z{uOOSTGu_&lp{`!*c{rK8(2l8J&n!ol7^td%E#~1kC4U|!5LT)GJNA1#m
zIq`Z_puks_tovdX--Jxc3bZ|i`dp)5P5b&iyK_>bto~zf;rF`HZnlNC1-25K@6!do
zFN^iH$J3CV!2KwKH#62VLSMU!KrRXnrM8=FH_LW9iJ%om^N
zj}|oEGp3p1d?yNgFA`b*L2hI9wZ1@w#Tob4Sl(`F4=cbp6a4s-^C>;2!U
zo3+UR1(sj5*RGkh2{Pohc&|aB^~QoR@nj-#JI$%SkDextdveE!i-Dqcf(Rv2WWfG8
zX|$}5L&Drmb4i*ZR*)x$kJyb
z?{$xELPpStzwpHz`NjJVS(*6+`6cX)5}vWc2f7YHn(6v^AIWSRmA&N<_|{
zp#`ki_p%ke^LT1tAq?1jyU-f3>)!ArKuM~hWT$V>cDn9r^*=3Y_yZ#+cEZ;jJIDc5FL-+Hqm^q6#JgH{7nq=i=kI2^1WW+OZF{~
zt!UX7U*XfEY1}TpX(u}SIwV22T>nyC!p^SoiC){53R|Ig6vwj9X4zX|so7Uy!`y%g
zhx(B9WQApEi)|bAo9%REufp;w8ZYoDvPI=RwZA1WdS{auPrg@dCHcM=7UY)&ijP@D
zy9RP#1Y@f1JnKH)_dZ5&uJ5D3HGmjDHs5Y8TLLrrP^DQ;zqQ4JJ8&r`PbL({VM1g5SkC4MoEHg{V>pt`Oor1)B;S&zE>)|9nl^e_5(fVZKv@J
zZ?XLcY{^Wa&dpaGZ`Oxu2R~JJd
zP<7JiK5|~A;pBTw2y7?uuK7gHS)n=PF(h5@-$n(p4HjX|
z80x1#3z^K)w$YKFslHI(`lee`H_b~%A?>hW6!M?&iGJMO_}L4Ea{Sm9R9HUpKTXGr
zSo*Lq>W#y;<-)%AcoB{p{swPBWBHgNG>xTM2IYB7U#Ld~N{mK}H#L>9UR*~r
zK%VsjaiHNnuIFJ0eD`@IDJ%fWPfF7RD7Cg1liwOc&x
zQmLsZy9K`TG0;>sCRH@?u^}OSfh%F3=B44*mkN9*(S*-^f-0jq@L8Gz|AC$mxQo#G
zH$>OYwdesp$iEdyHBf7yHre)F`Mlj19rXgVt)5K^BKP^_8W*%xr7vb3=5b
zv25Cw6<$BvsC$~~Yg=Y`7{~M3P>D$}GoVpbZMq{laH<_1I=2519&-g{X*hfsM^N@Z
zs&DH_9i4bSC8~@ala+1J^cY#yA|A@#4eTK8h}WL`@s(lm@qyup+JvUVRfe`l;jiej
ziIa)v1DAr~I3JUY*0$-Ru=uto!?M`4Dejgfv@EWkVm(>BC+>O#X!EuEPFNnok>@Di
z3DG4j4`C5JVD+y(9HljAYfsbOGjQ=vxi0w*tUU}Awe~da>)!S4zZhzwJ#Vc&0yoyO
zj;5Tfle?oV>t2HI)NZ=_Z&`QR@S{^&)_n!PuN}#nw5w@nS^pP~)#5Gdo`&BV^qsau
z?f16!L&xf8S^o&w+IO)_wyb*)&eLK6ZMQsha%Y-l-C>G-haNw$tosxE+i1~&f``1V
zOS~OREbA!=3JS9R4U6yREFPV;_H*Ps%d-AnkUeoT?8cq`ZFEh9>a+na_yln3Q=)nn
z@7xnN3V>FUZL}kh2tY^B+h|AN^P7!-hOmu}x(U1wKm(NC?i1j3fLH>wBZ<8LG&sd^
zC4naZ=!l%EJG``6Plnp}+^+WB@B6=t
zsadZUZF`B;4`ofL=kI(bifG+ud8nl=aj|c&w>2uXeo;H4!LO!W-3zC7Kzp=jmu&R6
z5j@#~Hm629`E0ahT@apAEmS7Ux>!`7Z?Cz*_ak;s+~039s)^*N>1(let4)71m~8s_
z@P{}3lS0ou&_i`6JHcegozV&CBG#U`QixK8c6En%GJxLG)g4aaP;XD%R7g@!XnPOA
zil#krmjF;du=?Ac7ZSq&s0#pW7a$gZx&Xlc3h>E|MzaHK7T^c~wK%|C0vrIK_H0{=
zps7hQUbiEP+w)O)Y5P~&v%zT3Y+OrrStUa>CxjirIZv7l4*3Szd
z+PC&RwLodlcLBoM^A&)w_S^*!)}DU?2y4%W0K(eydjO)+{cZKMf`+d5Y>K-LQQV#@
z;QiX|Il$DO2ikaehK}$56URflO1_!){5JITu|4NOG`u~(L^XyV4;>RnDwjesY<#8w
zgtccJKv;WzTCB9^I{;zr`4T`_d$s^XXwOFw#qIe&@cvuv+0xeUzcAl^I{%w#&)-5%
zAKP;+MElmB*FiF@J!b%fwdZ7ju=X4S5H>!C0ED$?G(cE;o|qS*J=?BC1DXzajq}-&
zI5L;#v!lb}+m0PJ5AUYn#h0NIG&S+{id{$AKlAT8{Q39yE
zGPb3MztRa~Uyu|ecTIn>FR8$#n>1$6nAglFHi?m5%Uy>Bm
zqh8d#@8Nz|XTiAV^r%;L?-9R#<*=Wk0#;7cPaP6+u_;Rrf2psD9Zr`j>W(jMfz<(G
zUot?kYEE*8{+>)?`F+Q+y_6}=l93t9T~5r#ls}92BKX?qT8oi&x$i21<<4fr`Z`Y?
z@*WxOS$c9ujOU`0JEA-ZmaY9Z_Sb{E_28b9JEA>FmM8jE`1yxqaO8BujA^#XwPV0`?!vagBA_f0B|SUa@V7Xl$nS7>=((gNzZ;UF4lr8
zjcIl5&vvY&T8?@R73Epz?QCC@Xi1Juv5XTnmF(^8STpn!%XeZ^a63jIoKIeh-s|oB
zV$Hx!u_-k7cGeESrGq|R|WovB8#@ON150F2R3${OA+DAhm3vH`V+ai8G
z-4Dmz+6Q#&etq12B)dVMw$~jGk9eM9;dS~r+~9o?dUBImQIK3qtL|+}tNRdY>^HYv
zAr*0%xwcNd?mgYQCIv-9y%r=T<6gAIGN{0Ux@}4Go;FIIM3gSTdQW$(Nz^yRCh0x_
z;=QN8Sd)mb5sdYW@}4%zn}qUGl4$Q~qr6EQV=d->Z|*NHK606?H0-5-NJMEztknVtfNa<#)qtXPyDjaaU%ZP>VeRS^=JufHNB&dof%{^u7W06`c*SE0
zSDy27A!f29bmsy*Pf~#U1HmzVBV{#ltky@w7Xq~a)gBCMNYzn$B
zO_A~cLfeF9YKvF&AY;-ohS6nWXlyD;!YvE*eqV_N_b+mj@bpDVQYvmYVN`*o%Au9g
zjrcb+$o^xECnj}^XEu)k%)OZ9F%{CJ5UN$`@0b+i6o^Tor%5nkhuAT@t3ynfW$qAZ
zh*3L4s6TvSNp7+1_D>W2Vr*_w8o*x5?tK7q5WN-}!qi2Ep&O-O(1wyx*GR@QM{O@)
z9E9|KYzWK==>;IY06pn7B@5~GNWHuLKQQ|6*pj3)==ybR06`<~!FoUB=T?*8xNP=77$)3FxKD<|
zm4poc8PB{#Y~%i9c)%Hk33n3ili_eBA;VYoWEjfbr;$+UT!cFb_sMX$l91t@crqqp
ziBX;I)2J%NFyT(ZeKH)bBxHD2PlnNf+^5kw6vKo&3HQlxxRQ|J19-3{Vu>+i+@~=d
z6vKo&3HQlxxRQ|JMLikDB;h`d2|zJSxRY?742LTT8U6=7P7`V7!nEiK64#m>`0QPH!LQ80Nelcuz9tQCenw_Ux
z_6yOFi5OGax;GFn3}D|^141-p;2}J|(sKzCg#qmQYCwpF4A^=#fM!4yVc%B+LNsKc
z70d%8kV%bS`7EO<11G^lm#`iP?5_Pqxix
zA|~5}<>^fCw!@W}ZJ#*aL%Gc-BPQE~0>>V+k8r5vQ1c?*Ys{XT#4EC9X;7LpOu(w6PBkoz1t2~VzwRClWnYSp|cy4ZNl>G
zrgz)nO3b#`zaI=1&zrySI$Kd-1um@fbDw(SsNW;TviB#)P||Jr$=3!2Qss9|+XNZPT4aevv!rIOTx?Qm0FW8IQaN>5W+9
zV92{COWeo9tsGqT_U*-;D?e`A;DtxJb?1zoCNv~oV@`Y-rKt6bG{bq0n1bSyoWnp4!rz69{IOyeQDf&d9xD5
zij?u+8#&&MT)&Us-r%ky0rQkD)7F)jJ^Ph2arm>Jd16zZ{qi-xw>@ZC|0_t~qAZ@h^$TBTYyInV
zgAPYmD{*}n^4JZD}P;n;;qSRoPXDT2gTD+l8~-f+tki7Dgu)}cvo3o{dMrFbUdZr*UqR>C`xDa%uS
zJ!*%Eojj;}4+TvCLZ}^61w4aX$_i*-c}ceg`gT}>+UpM?OOc9&?Bd==RS(6QAc{5X
zjAC6Xij_gd`j#ojy_X3Ex^Ch8z{xl?RO31H?5nrqoZ8Xw4?%nLh&|^NE=%)>J?BYW
zu+@F%9G$3XH{5Iw9`1USMm+zI_qivD@s?%TigEI4r9TRr?mNBo$Q}B({m6M%{c7%1
z^-HPuuE9NF9$a{tRrj`HTx5hy5c%+~9c0H#5;UGu}5d
z7Pr8WRo~16-^~8LnK2c>V%znO^(R3$Wga`O$#)YEMo$-#b%a2B(Q;*Ix
z%Z3JYHp?KYE8OFK4|fHnw-t4D5{6e!(0x1bcmZC=dCi@&>zzAN{ZVv(d|d2wBucCM
zj5_(CtXBNm$)rj8JSk1`eZR_{}2`+gZi``
zE3d>Wz{BxG!V%oeGzvnw9oE%mIJXn_lj}Z1Ch48Y^meiD*x7B;QC&%eufp=4OppAl
zD&%rDljfJkQB%>Ff!E;2(nU+vMaX{OeR>*zm#PoDI{v8f@MEd+T9Gt-saiz{D^UuT
zDw-5SE0r(qCPbMRszQVV`(PFem5t(VfwK2#_b%;@1ip>uWNx_Lw#e$Y1^0TAXm=s)
zE)V*TUEw|5ul6$UX^ZD#Z|8{GG;ilf&w1Xx(Z^D}r;}?3c~7T!5-HdpBK6Th|IWbq
zqAUru*{Ydg!ptIiWigRnWE|q{i1Ca#hUWnK)t&F{oL--NY`FLIu$6-LeKjb(+g5x6*Qi6dpi2QcUqKZmat_#X?-d12bf3H5=1RR8AjoEDSj6P
z@L;Isz929G51hD11hyk|Y)Bx7U}E666p9NBMjR@^`&I19aoB5~ezP~lo;a40zsZ(L
zm3fQ(ZL-DN_K0t8+HPKqm&@@+M@MXJh4rNI&K+J{@$H~zT4sAYqdivzTYQIF
z-cJecu>9dU@+Gc45=3o?@Y;9iNt57CykzuzOCZkM+8>(gM}F;Pb0r4?edVuPg6L?lOi0AgzJQ*RM8{
zUL%3Or7gAP@Iad{Kb~GA5g^$B$-Vx~A+3yg%^%71aM%MeQu+
z13XdMQ!OYa-pM$Keb7$_dlv5t(m%xF^~1j<9(=gP`?%yoE#8Nv|2&KLC-4`|n~y3H
z@8}@YH(bBSvVJ`<5{`7Q5sIVSD-k-^eQVHPG%xgItY!N9-6aqi<(>l?
zW5Y
z7x&==@rq37Y0!$M;(Xu!JnOzZ>q~joH>fMEJ&jLvRCz92d%Db4nClt4_VXP4gLuy+
zYd>F@Z%gx>kML^zoBok&KcC}RH4|fKq|Li8>R1Z)(Z`apk2y9F``BZ#*vD}d)9Z^n
zT8_ng-?7+Q-XCg17euAlq7K;*(?X)}SXu&!7q=4~uv1?j^u*-)Ec*JBM629PTgSu7XP^7aQRQ;C8?ric^-<
z$AiIixZB_^fvazYJlp`>gK(EW5e&xS6y;&Km&4r)Hw$h&{!zvvxU1ma2R8us?{F94
zV%AWcRBVQu4tE>eT)3&*kw3UP+y=NC;cljQxLe@vg-hQ(4Zv-M8~DEY%C*b!
zJBnW`;(iR*&|^zV-4J6-O7q4>yPyg3(3DSAl7-(;l)vaP{A&z4m4)5!MrS9bHpXNp
zrQH*on>2PqTuxHDx8Ka9{n7IVCZ&Tw6C@xBEjqp>jbw|E_9qyZo8c!NjnRcksrSU>
zlg<33^yspINolzWDUK?@z%Rz^)~LC#LHln+b0s+WBGdrNq0+ofj&1E|nXNK#weh
z3an{#WkN^VH<4DfW8}2@!0^OqX(Rncq))_rXsF`&eGK8R>f%ED<{H
zA$ifC&^PX(KD;4zPLeD7mZUUvYDAz5QaYN)s~|s<@@Gz)
z1xEQZvl1}MpPA7*_!EVF8*H<}n!xT;*e}7Fz!F97kTC6DkLwVhhIrZT6hr(ctUrwa`r~5mkB_sw%3lB<3q6!y
z$S2Q+Jo%5ZJc=(is=p2}Di8fHAIg&#b1mhGZ_e|1Q4^=q{^Rb_Ub|#spAd`t>
zr=Db5A#)VRQ9a4fu}u?>nR=2L3z@5MJhNNL4_3hVxtyN@$OLfA(vyq}GWX$lMwOA}
zqPo5hGFdp*k@ZjQ^)T57>#n_aAuin{PdW~QU2bAPXScI8@E@xI`Q1+Wolf<;oHT4`
zqi>DUCc|5l@H0x7zCB7C-eDHo%N&+#fjHPij2IViJ@tm%xrU?3?~97#3@{oSw8>*3
z`8`pZr)k5m5!+{98HZv^aGXS&9E%ULVN9Q5OT&6f8%{68mdf#*&Cs{I;gP(93nufK#t}F@5*y7Gw0*^glB-xg$YE)>ox^@kaGJxq
zk^H~DZ~bZaCa0tudFJ`JGz%U0V4aTKU~s`JGw$U0L}ZS^3>q`JLESaiV}b{5Kl<<#%7@
zcV4A^`5jmJ-B$UXR{34lDNBs>_p{rK#iB{x-YvJE!uy
zrt&+cw>u2EQ|!v`lFILp%I}WK?~KatipuYZmMu5Z%kPBB?}EzjfXeTF%I|#2?|K@k
zG);cbQ-s9`gZW@b#(a6LL^Xnu55%B+}}
zVY3k~h>1xljEPE}6CE>b32ars?lQQ`p??+9u7dprl*aaIcLGRG;2CO5N=J&NYAjQTPLQcjh*i(Pt!|19PuryRh%^UT)Qz?
z1vBM@Y!Z#pqBXhhm@xhoOpGSin{Xbh$#p55$7ynY59j?fxh{nB{+e80!g;(V*P(EZ
zRfp%#a6V8=>y{^I^7tTJJ~6azhjXmz4Q68a@ilR6Y`5H^t-7;2`Q*@XfH^is8>Gqg
zKb#NNG<8-^3%?s$rp{`2nmaP8q=cE{za-W8V&Z!G?C
zZWl3pbX*mUFQ>|Su3`bw(4G86Ebmv=my4OVD?F3AJibFvY;&-W(z^0b`Jn9v{Gz(4
z=e6LJPkG!(va~hg7lorVdE6!-cE821yZmpD!2RH)zjYEk*u=F=xNqXuot~o+_(u`=
zsR;aw2>cvWNOyKG0FTnT`U4+MYLWEdpUQTt=RR=yV(#X-?@!Fr6)qmkf!$n%|CQyJD0~O=GKKG9-k|VTnQu|}5$10zJiz>f!armlJukdm
zbS;?bZJ5G4nU7UCI+c(guke1%XDghpDU*Je!Ur?=Dm<0>&lFBiB$A$O3LnFKkHRlw
zeo*1!UIz3WRrnN^4=DU9=Isj4W}ZBs^3}D`b#TgOy29r$&s2B`^Cb$uf%!UxFJb;e
zg;y~DwZf~IKcVni=6e;siuvCa{vGD63crha693CQSrd)SFHraom}e;b0p@uM{~2>#
z;lEJ#{7=l~
z^#Q3z{HJ~N!+R9@7M4G#@aLGz>j+ZMi_8OxJl(&eem8{CArouV|q^f2#03EHAI8Nj)zxkHR+(%-bu>s}=q>b9p^Zrg?|?ZHoMH=I<)}
zGv@Mooz&CG{4+&9CXOT$wEUZ;q~Rtqm)G@V>>%a~75Nd&A5r*4%;ohzsb?&6`izHp
zvocSxo2jNvXD+W#%Ghg}rz-L~^D2eUV=k|6N<9miuU6#k%>Slvx*tyUCa;f5J=M$)
zDf0EqCoGYYhP#IO6os#2ex<@2ncu7MA2GjQ;s3+@Ckm%~@|6E}h5wFuhr<8J{7Z#D
z$$VZZ)S0$j%;oi98MTl3&5Ha>%>StHH<-)o$x=@%bH5_r#(d~fDQUPLGndz+W$Y>D
z7bx=LLjb6sI)x7qH;xiCc^zBoNn(DdB0rq@s|rtJF0X@2Jr^^7N0Fbvd}^7LG~B7o
z<#lu!OV2h?`{pY0dCWH}d=7JY9bW1wW-hP4OHR)-lKz0Ar1C{w
zd5$8#iut_?U&~xx&zE{OFn>sqznA%^3g66J-fxh4e#$)7A&KGshPg}Ok1?0`D`cvz
z%o`MWKl9@Xe~!7l-y!w9$ovaM{&nWdE2O01zQbJJCy}uM=BpI>kC`7;_!rFOeHN)F
zO5Cr+I;hAeFkiAvN*eAk=JGy`j2*$;rO1zA{-(kwFqijtq@Ib)k1O(5Ghf8_fu#IQ
z=JNiK
z^SugR0^Z$q<0piCg4V3$Q(o_VLgDhd?#Bw3*J-E98wH%*J*?lQ@E4f>P~iud?^k#$
z^Hzl)XWprBc|BoFb$GqW>j|?IF0Ut4DO_Gp_^!g`^@Lw5`~>IcQH7smzE$C$GyjXi
z#n&Ow-=0&ryk7B!!s7-|ITN(E6+V#pVTGqK|6JkcF~6WDyx!87=P3MQ<_?8l#{3S2
zU%|Xl;ggs@sPJjbf2r_mnEyfHIn34j=5oCMQ;}aF3YYgsTnd-tt4ZOnu$~7Meu%kW;cd*1D*RLCpDO$`^G=0-#auoQ
zAZsI9JiwHo$>#_p@5fv|e<1ll=JL4($&;AN=N%*;%v?T?AbARN`5c4fBbdwQA0$7Y
zxqNOy@(Y>E=PM+?gt>ekLh{R)%jYO0pTJx`e<67WbNSqc;ETt;~ns8lL}p
z=2;5Ao%w8qH!xqU@b%0a6n;1J?<)LW=IZk;cXXl(F_v8eV%!$iIMju=4lFlo%vXWw@E!w;Srt8!zrG(c@(P+q!W1x(NFJ#_}=s
z;qosrAE5Bpm|GP7Ci7tmKg|3*h0FWJqZR%x%P&y)G3GZZyq&p2;h!G`haY(rkWy94i7a9T(1T48XxpCtI1&kfFGJzKI2
z$v94M3+q{-=vl*DQ;w@QMd0@_Z&LI$gGY7qeBf4=SC8}k%+=?$USzJGUms$w+I@$)
z`kd7XaC&}k3-^Dy9!6P=dTuB*Dn`7w1n)3#s%P~)&vQP;^6L5ZQ_S0y>R%%KcZ&)#J~>;s7K;8>;BPgt_`$oRztHe0~*kzcOFt3V!CiH;3ib
z=RK|mr*W4x!^pWDcjYXvK3C&n`64c_><`~#-kNXdk^Sl+)|0N3_i>hQ$TH-mo>!T>
za>Mg;1f2S_U+K>uv%H`6h<6u}wErNZT*-R4o*~SeSzg+m$h<)**8=8Cn9KgW8l1+9
z+V1O_tJfv&1CP?qeBSt{tbfaFBcHO~wg~y*+BT)UE#QM;x8WK?UR;)k_bcYQE#9>D(?coV5kaw{W>GXTCvj4w#(H5%_N-@TXaSn$jOmN01*hto!~jMsW12X+~zI
zaGq{t?o#ys5IoJ4Z|Qw3g8t_t@V6r9IR$>9RBCdEq!{&_#r-gy<7P1TEA4Wd;3fs^
z{Q=9nvJ8M}L~Kt)kbfZpe~0y?O)}CWD7MkVjeKt5{^MePgWw!6Ikge^U96`_ssCq~
zCr>q$G_bDYl5;ycnGeN*8qFJ7+z-bww=v((Csce`psixQg}Iyu9uu4so1EPd_#xKg
z;`X|L)5M-@jNeZp=!qM_^~3G>W43V(
zbJr9DY-a8h+>{aa-o^4oSq4aBzLohBrCc8}Z_YFXt5`k-6Ckx$@|6aV^)r)sI+yoe
zmak*Jh4qWmCB!`dpkspoqT*30?%>7pyU?%hLG0#%U^*r--
z&LFGPJ`tP~o1FgV8+QGiPc)y{Mhb2cWUrOw(>Xu)FfWZD?}@D!kYCHbDZ0%~TqPDc!VK1+%u3n8KrjQ+yo-om+t33|8yRv45$YG7c
zgUl^2ttqc^hmU5LQK2DZeMxDEv5CP-@3KV
z;r5g-{Z?63C5qK8D|6O5u5j0uT~S?HS#w3%74GGw0HyBgs>-ETR6DB6T&t%{nt-rY
zke;57oi$xEux;U@$<~bQ>$0>0E5&4p9ij^+lW(#jI3tVGEy$QWB}>bmn4XbN`-x&d
zN$e+!{S?}hN?q=u3Vz7z6Uhqpf@By38uW=|GhfJKkJwqlkGc@CZCj;0%f35!fcMEag;q1kM&%mU5~rWv;-olv8CX^8}WqoGME>Ltt6T
zsiKs5T(Oxl-{i^AfJMb-%6yYYU{SG|GT-D8SX6AL%r|)i78RQ*^GzOsMa5>ye3M6D
zQL&ja-{cWkRBWcqH+cjW6`Ltrhdcs{ip`X*Lmq)e#ZHr@B#*$NVyDScl1E@svD0KJ
z$s@3+*lDtq@-Z;sJCM$y6rTXdGd%jQM=P*=E)^x6;mx0(`;N!RaH!rdNCy|rZmN5tXM8$)h=`O
z8@UTr$~StLfa8v?IzYfw2ehOV(3)2;*{Fz6v{4a85DOq3T4a-Sp>+y=8CoGGzCw)J
zLKLjfIvGDJ{8RCph#&k`#34~3225c(I1fos)c&!A<06LEF_^q5-OysH6+b-Y4lS47=~7PdAE@iPno=4Dj`B)(
zMJXNqa2M5v8$d|Fjss+8ZMjx%U*TAd
zy-*96#H!(DxST7`6lX2SPOYOtbFQ#+cG%5ecOtu!>{?A-wNULgDHEBqtDwqJv&^$x
z%cX&qO9L%86TK#vMw_VjIgXWu4$pFDIga)+!KgdsQ%}kdiOj7kXOUSHJBvEoEQ&>g
zal~v2eshCUo+NNX<>%AcM6m;t?u^P>x2L4C+M(qa=grCe25pwP>O8AmvTx+j;Lo8$
zyqd2d`USi8G$g}gH$J9qY
zsFahS2pZG9p^j9TMP_K`T#{A$4c190rJ-SAc42--eV2NicFmx&WhklahImnZWp$TH
zDLm0Ej0}Y8VRn6Sy1g1tgm<$tqf$!>bGYI~c(R61)=Yjh^0FFG
zQJKXwo~f8};mf{GE5^uo-~=$c(v$C=Ato>(;#$3ctH!mOGlFX9Y7=g!aAs5?ollYu
z#0mR3uDTL*J}uPK*?@?&-!KK$x@c~Zs7|zzIQu%IG7%A~Owmoy9@bEMWeD#v
z)CpK1)CWSH99c5z)Q~{a*>kU-V=K(hp{z~})dls<@Y0ZBZ5A#+*ejiOz5;>kGjs*Q
zj;kXpowY0Mc`jT7v6YoM+-_&B9iup}Xl_Bl+&Ov0`rMM<63;=&-L%{p8O+L>S&*Nd
zW6!|{1av-X$mtJ&6E@{T(0wM!$G_ljAG!71_(5X$inc7=jP;t-;Dk_)N)jH-lYqF{9
zRMsr(E!`QpGlu4Z63w1pQiyAi^0F%y2D<2qqg@=wRn$6AS>5#>iV3eCp{H9dc6-^X
zQtDJV#=T8R)k~#v>xG79iKnW!l2Q_M`cQ`x>y?9MCVLs0-s7-#<$GErWz@^(y6rez
zsc~Xh)s*9kZdWpzkyxXrstZq?kkOMqx}xm1=T@R;d9vXz#>HiOC9YxT;&N`SbM?QJ
zHb)eRj+nabU_4=T<&m=9OPzP$Rr2TbmQJ)4dZQR@eGzs!9DSPgFnWq>$}w^DrZha~
zqJ^$2t^Atvi%7=Rius-9xbW(uipGI<*!t8}yY%
z?QpnoNvR&ozGzANO7wuP9P~0qG3LwaYHKko(;3v9(i*3`w-!620mRIj*QethT4Z|D
zFM9;_23quc?Gcm!)0MH@e$DKQ1!;yj2E%b=SGCc!QP#~W*-O7aqi)bS3vnVHAsNrQ
zrWPoyEUWFk38=)>(Mp`f=!n(WPh~pt=uiSJ7olp^k~=&L%3Zw)hi48Gf-oW$)IONG#aUS+#=l27^6c(5
zDjk)bc@WkIE67#X$PhK5^jbPVq5}dt*bR>|j^4W%t~*wg)tFcGKCG71*3ixil(>)l
z^{SN)M`hz_6Wpt-ag_?Lhc2bTT`qk!PLE^4vYNUHOY17D%D+`vu89y%*_LY)%2(GQ
zrQv#NjTl^Pabw~S3)m4?i=$vlz&=-%N1H&mzqJV-$13dUt~uhIbP-RR;8@Q0$(NTS
ztqd6{j2q@g3hAd*WC99Ac1o)&%a8>p6k|6*STi&)b-T3*xQS191mpt(|DKD-O~n65
zk7hOU|L00~jG{hP2~R-yB>vaAOmF-@L3|&FfuCQbk;1
zgGC!*QYF*N=W(*YNViNc|8K8!<^PX|h}bwgGAO=#hJQsI%D?>ozS7;wk~06&j&v6w
zp8hwy49oumEZqi9Px&VqnSaJwIXp6snEZdi(v|;@UfP%GW%+A3eKsY=CjYOnbmjlU
zr}TKFFJxb)Uj;^WBA=rSJ(t4Ir^x4PAR;!(t9f%HU{0UL>7;A?f6T5F$?TQxca-$<
z|LjWlg50jO1#L{R=^dij5z5B8E?>U$9$>U%PM>raiCR!%P~R<@h8E6a5d
z;%58u|3S+vP;$5Nzelj)@UF5=AT<^Q2h|DKV^BMZhl
zWWJ@!Q
xX&z`tB57aFA2hyX-VI4(zxR2=ywy3O

literal 62048
zcmeI53wTu3)%VZjf|0~bKnRE^qk@83%!FIOTP7EnKmw5@h@eiA$poTFCeBO(*ebyR
z%ZPbRTWi(YHv0Dc^wrv8YmuVT1dxl`3V1`bHQ;SxP@`1{5y|&o`>Z`Vvy(|;Yx{iP
z^KkOa*?a%?+H39mKG&Sp*}1c$OeRfk(b`oSh0;@XlFkvu?d*Y*uBBZbH1yrE5$#1X0ZmdVKRQ!rj*>
z5?d#`GF=oHp;k`U%IT!5kpOM%#_1uA@<_il=BhiM<)llpwB>)`DpFe8V5AL}teoOwH_f#G^
z-Tjly=zA##6Y!%hK=pDue#5S*(bCrsT^pxGXE!6fR?9cV*k86Z);A6tT0f*XDJJE7
zOVUu7iPGYtw6pS)(#A){O&q3~hHK8`=ycLG9={9mn~2{e{4U1tQvAM$-(>tQ!;d!Z
z!rD|ls7-@A6Tev;z8vn=_@(2Qg&%dfIr!P}qb&!&T;bESJa!kry@vfoaIeMhl9R7C
zjJRb?((AYW^4L2m&zC)r_4->sKi#=;z-^zggc=
zdcl8nY<_n3%qb(6{BPp=Sx0vzKgE7oaN1dKy?)UL
zhwl6B@5_sRSAOT?H6x}t?`>G-y!ZJ@EpNQD{?g0$UU%>3pC(@Q(C5E=qUNH*SKKvr
zne)USA8lOZKpsr>|8?#7K<61Bzqj+PJ(K=%%Qb(!g1JzDeAvRm&;
zxb(L1zfHU;_hiOB)2G~c@8_+zy)x@h4WB1IeoM)D?`>O7RUC2KhnV4g#3o}<_r<57
zAN9qlKUwQ=Nhwe-M&k^L0qht3aKQ=V1ilDzeg8UyM19r6C%q`SWv6bqM<`nIYeDdV`AoCKql>Zv82Q%{k
z^Lm9J677QgEA{Xq>o1Wj9g@S}1yD*0c{`A=fI_XufC`-EVCX<%Nj@Mf-`B&FUm+3sGpEBoQsTn`#|gnG8Sne&sv
z@{^c%a6f6}ep1PNI`?OblAl)Q`3m2``EO*Nzft%>zFvAmV_{D|8(h2`a8?kq8H#B2Fn&+<5|iR-78>yx9km0X`2
zxgU;U{ZC4}%DC9Uc8l5Wc`WZ{{mraj_VY_vek1q)IV?Y%+r5d~@lxiL-{Ev5Ckw>9
zf%CbS^LZ`v511FTo^2ZC>IGig>NI^+u#jP%FZb@B~$cJ`;Xbd}HLb!1g|kc|xZ3o5E_c2rc=
zRAwTv&!tuA8p=Jceyv7SlZ66%+^hbnEM&OdH=}7>9#6H~fpV1iD%{nY*HwwUYRgw;Y
zU!~@(bbBM#iJHEotjv?5Is0u6V;ZXO0$8(&bb~0>P40^7NR@0<>~&R_4Mq8m8
zH7?i9gOPK(tII2v)p}g>-PIXr%bP2zmubsOs>>=}*%j43Pfm5FySkbhb`XVyvcZa?
zwF|1tGHS~QqZfT@@WRlHJ~7{&<*F=+*uRZc5AE42+_jZurBwdG8o&-czrq5m|)9G
zoP!%xhJ=L}j)Ukn)b|VBm@cZzd;?Zyc8$BT(&lu!yk57*;i3>viNf!2iuutxklnu2
zAh)bz;C%Oy{@M~OPP&2>PZ~5Rg(0m=rdBmd9VdLzIN*uR|V~~gkF(kJDLbVz-^SF9e;JTtdO=6s!vQoFN|{Gb60c$<4{gbfivBojfh%w@wXtr%X-_`DccF;v6Fi
zS8#FCA@;hT_H>>lc?|9V^u$Dkg@)jYL=^5W{OA7k6BDvmiE}hM&#~ZFJEj6BJ43X`
zz-TiemyH}x;d_5{AI=O<)1GGLIg8|DwLf$Ctcr>==uW^(93CaY(c0@A*7Zk)^vnAa
zIL#GXF<(#E{R=%Txx7EKSK$_xKcH|KPI)+7k5;IM(-eL_pTMOn{O8Q=3Lmjp*ORaC
zq(wSktndbwFIM=F>vZ|$3SZ5}w@H!57#|7L~%;|5*6S>c5@>U_7t|Hyo=!uPV>0}4-e=z3ZezJhsB
z;Rl#^DBQ~V>{R&7VqL%Xi0;aM`*Y@T3O~U-N#Q?c{i79rLy4}}qVTQECn|hgsV<+Q
z@OtKH3V+wB%cm=RX_?OL3V)M%zQT8u>hg;fUf|MsvBFmf}G
z8aNij(AHt#wC2jK)4+#G2zPXo9>C>Zti?T>rAhY_kLtnNL<5gD@Du~brOwcnX5hFq
z8rsqgoXRFQdz0=XyY!P=ev|GaCi@DJ#Z9!=WpSVo+KQWWFUjLld}v$VqV=)^nqQ8}GZ_Z{TO@NyWa&z(*MP
zV+KCbz&9GWyoW;3n+<#v0p6OMbPwfIUb`l;yNULCdC$_lV!yXZ50LyA-7EG7n)Cp1
zc@Kq1YZL8tJ!5sR*aw^R0Lh=Dd&Rz^Ne>Xu*S%uj*`x=EpR0TIJ+F_%#~FB>fm;kb
z$-vJu@X-c-zJZ_eo~nUUHE^m1PSwDv8aP!0r)uC-4VeM9W)
zxC~%l*X)Z8&O-{je@EPQJ!SW_!-(ph@ILnDap~|;SX|_3YwgB1;UI-gkZ#*6!uu&q
zr{HbPBK#tS>D0SzqX=)KFr9L@HHq*R3dd5oQG|a_VLHWbs~6$NDNLtLZ8akN2!-jC
zx~*7*AEYpyO1I^U@Q*1>r_gQbB77Hx>D0L`MTGC5Fr6~DSw#3&3ZFsYBoX#dm`if{&n=@hxG
zNrY!nm`;t`8bvsj!gNa9RxiRAQo^SFBCP20{ch(n))u!`|}g)%KRoXuD~recwsn2q^fVQdbby-CAxQm(`QL{>odBPOH7F$nL*#
zC!~EJ;6j_Tj8InZ3^sxS4CByIMLmm6WBq8y~6JQ+Ooxh
zSi3pnpgr(z@Lfcf+s)O@<>ty}n|WBwc#<1uH*L4=%o;yh%L!aRK8d_zO;p#pf!3wA
z>$m3yKFYSfAN&G|+k-{+4Y$VG1Fr|)L(m@h?1r|3q%>3jTVavNOW;NOhIvW$4bjNG
zebP?*%rEM$wQu--ob5T93ha|Q?URn%125PcVut-GU62lYLw>j2{QQ2q|BI_9?aO)f
z{hT!?(8E@*xFk35VbDpP3Avq`W7=U4WX9P|!Q6l|$sUMadIK^kE70~F>T|XIO4`@$
z+L<}pWNmK$8-8!;?PgnKE3y^Z0$&ydzA3P;IbuO}fktu&(T4Ta+^_P};)vpw)RTIR$*>`&x2+
sT%N23{eu?vt#Bs;dLRauz;ld#Kb@WpDjmT!jG{2yMO=lsMrY3F9Doh&GS
zwKX@eXYT1)M=WzsUwecM1P^s}cdu){y>{*f9psw-yTh}3?gq$^*X+Lvg|;{3j*lY~
z30r6m4ScqhH15h8FD^2Q+UZ6pfg*#BuSuhMT`Ur2Z73Tbm%Z+N^V(ym^sK-Ix@=n;
zLgsOr9x|^xh~Tbh3~{aP6?hx_w->VXmB@ST6C02bG~#~;qT7EZ#t~X$Gv!5OlN#o=
zEi_jbWCcD7yo4T|l-p1_K7sU5IdTJS!C6Ap%0+rQ5E8wf%S(^vYQa&u$dq0pw2bH?
zUmmBTuS`NePon0g)<#=%IopGw{!}P(_GenK3IeZM(L0Zf4laTLTVMxTBWCT}zIZ4Z
ztt;6c*tLbO=USV$iW>ez&&edJrUX4ZTSyi1EE`4?ELbM;v+9qC(5pJQzZ2u_e~LuM
zqqj@6!0QEpm&|_=!+Zkt3Ppjp$^%J(PuZ`c{
zGd|I4+a|(RC?3Tz@3EP8mz%5il-n>jV8Wq3WIbALE^V=Grhc>K0_@7oZ=msl&mvn?
z-eY@Pg5$O~it!Y9-By?rc)2LY87ycwi*^lW!U)Dx?b+5n_Q1y&!C8UNf>!}z{MZ6J
z{hi&ux9#iRUNs_k7GSpZs6Q>EZVlBXLq+B>DQi(DR~JJd_=BT*_mT4|4X41HLSPGt_sl19&I-*TPa)~L
z=FLI?O4-*C(54c8>0kPb-d
zh5P_M(U1EYKL@_*89%n7a`R`+TXDdmVd=xdXm1#~I}7&xNAhvp@HcpK8_LF?Nz<6c
zJS=PcM0+4>yaj8jwV8rxc9hSK)HFv=6gig#4>o^^x)Srb7(iHUgP#DcYqsxdrdqJe
zYRDI31yzdiQHah%JyOhA!6%VbF;T4^f(Z)iq%uF{i8@{NUeC0%6c*ZFRoW9XgqWp8
z^;)1^RIl}D@JY%dwS#d!3A|}NY7;Bj`rPs6E$6YkIk;Nn-u%;MN@CvBeCz8>94^jAVu@UsK8rPtDj~Cjs>3qLoLyq^e(2JRyp-xJrKng#i4O?dU*Y8
zrru+yugz)UVI03^KqUr!TF^pOwc)m;;3)FYv1}qd#tKT;uw3SRZs>1R-`1n{z(+w3
zu)QK?e0qjivq#IS7V%K_VQ@bwMZEUngO`WJ#|1A!)CM#ix_8^NWU|L54xV2M&H%%4
zEe4g=_C!ZmeA`Quo6vOMxFPn|5?TgUO}8E`*cDrW0BwPez~|;iaHMrk;B(O>&5vNx
z-)C)JbI7FCYio|v-ZymVb~z6R_N_StWLk5a_O%}d_D+DBDBs`K9EKZXUi%W9^rJgX
z=C%9bJGPVVqMO$q*Zt^}=Cvo`4|E_|BfT8esF~OOiDR`m^V+TOTe}0t&8EHn)*EEGKAZ!PwB7Ms^m
z5)>3<-P`8C!z><^zUFJ>Jl(wR9*|wJi;DF-=9}r9pX#(8F8Ec&q#9~zw{-ii*zW<*
z%(%JH$2B$s7&UyF{rt~ZexS*MeMHF_ixmziR7s1Jy@&MroRzPHvK~Q!<+tDq30p!
zp}LcuU>f9lbppCbvn%!%h*E|2bcZ-HfZo&79ggDAW>@SSNK#K|`v}4E#$B<~0jLXD
zo7-Lz5)%NZ3jk~pU<3el0f7G%AR2&L9N>NdzPLeec7PuWa0r0fvuzE6h9<>$eGyUI
zp3lNd+dtEu^?G|I{Ri4}1hPpz{$Fj+m+`FOAlvg{hz_njS3okXJ<9>Y+LIow3~SFU
zfUx$Q0ua`o=L66P4z=enfC%l`wKUY8?eNm}FSKX;f1o|Hkj?L+Jx?qdbbIcD=-}G(
zQAmch=Y0TS?YRyhtUY}IVeRPz2y4%U0AcNE1BlR`7bA+t=h^W7v+b$rKI0)(~a9|6MJ^Va}j?fDacu=czIAgn!a0f^9^OAy8F
zIS<}%-JWq+KS?#UXW+onmf+d{?)CH6c!vJF7@yZekFh;3?rZ%VDb`ObL+LU@L}!w0YF%Lz6KE1p4$P!+Vl4SVeR=afT;B5wmMotLkGMYV(&r}x99ia{nqVy
zx}iPywej!_oqzm0j)z_=`fl3uZs-|gdzL^nygk1`HHIG#wTmN_8ITNX&kF&<+H)j8
zSbN3*gtg~C@RCSadmaV|YtMZE5!!PLqPRVufcIZ&&z80!|AF~72HE^B#^-YdgKp0s
zL3D8KxeStF?YRgbtUa>X!RcV5bSeSpOzcgy}eQ2
zN)Mg16UII_v76l0r-^;88CM@dV!E(BurDWY-0nY~UiT~en&b7zYTZ_mHc{*g6T9uE
zSM2_eaDS&OchZY?(;IgG;b!~gBcDWhS4^=VJ1FF0l1mSLV_zLJiY`^u9w}{s)zihk
z@N~thG0AQAk7N?_Z#&y}Q>HjedS=W&bYnKA{F(h1z}H6CTJ)^TeNPe0cQhe3&~@yf
z|L{0p>CtV`z6*|SGx_4pn})1E&ECD!-o5MSwkThs`I#Z*fz#~%owzx$E9&UB7+=DM
zAp*nz#2nog=S#G27-AO)S6_C;9o?3MWJyEJlPDSPk?cycPuho@j&}c{R=er1R7t3v
z%g?xPAua;^unF~OKlT=`4vh1S3v^8CoX~C7a19`lBA-B{%sku|P9()7zHnkvjM?sQ
zvD9|F(7A$Y+4LqV%D2eh)v-FkoD`F6o+N52$=}tvdc;4>x5p&o_Jcq;pS&8q*WYzw
z_0SD5$pHal{9PxJ@EM5dAj;p>z4|osrkLdQF{7v-Ab%nkZ2uBz9}R&lw5>#Ki};0f
z{}*>BAFx~Z+9&NrvL*JJyS;Jni01_sU2C6&8=9{`PgY_p3X)}RwfoyFwVy(b1M#vyP^-3@(I+~n2(7rkYwi&J;zM)_Yfq;zwg){Q@$YI6+y`nk8wV`LD{jMI
ze#U$AF_R^tJLfjUzLJN#7Tx2U^^grQ6ylz%JLZY#u@kAsMrxgWy>%vr4_eG9G0Et@
zG)2b!6KxZksm*@TgY-#9A4V68p|PPb5f37u_Xi5ictj~H5l46=V)490CO*9c}#^gDTHd3`a32CIR#=;=x-8?*dg}k_H>A=<{3Lg
z3S!g_5$X^BFekN`cQ(%y{bFKPq6J{LdFLJgIf&j24PojcBhihLF=#`{sB0wQ8Jo73
zF%Cj{zt9Edhx7uFUV#4e8j^+d`lQ~S%|Frm@WjGI3v~U&ymLRuL1bp1$o8ZJ%shqW
zB=bbvkixRGuaBHlIu;ID7!C_~%p)cdJB$(RVE}tPY~np&!tS`a+}z$$ZeAS&k9!-)
z+l0Fa9|XQ$Ujb_W9A@wwQkM*Ms*;f52k4=jNQR@xFtVf=Cfr4MP=>>mgbZ8yGaOBZ
z&*){Ca2Mf084gzxGW@8xg&m=)W65xQFT;ep2oK6|xRQ|J$^97~LWYO-GEBIO@SqHb
zD+w9?BcAby*v6-k;nRB=Cfr4MP=>>mgbZKNpJ6EX9!El@a}n+$JSfBAN2
zK^YEL5;9y4!;Ov7bfK87uNSj5o_%s}FE`IJAB>^>KI~VIfR@ng{7TsDJQCuCG&|2S
z?-imU6VZmU^=%+d7{Gq8283wHz$18EqyG{l2m{y;)_@QV8L$m#0L_3Z!hWy@glNb>
zE1v7<4Q=h=vTDJD>qf3NV2EU=0Y-kb&p%;6?vc
zgy{|jupg`eAsRAp{eT8Ak;4G?gEb&TLk7OalOX+<01E&NU_V#`LNsLHK^PcdC}XCF
z1?&fFL5PMdqzzaE%;+$H{a_6U(U5^R>A4S{2m78YFmuBI_JcJbL_-GZ1~h;f8V0Z*
ztN|e!GB9#L1DHc$fM(J`PNzaNWZ-E$>JmCHFo&O?hh}8++Ls}Q6Abe`Yjyc#hzcEa
z4qs!`5jy!GU18@ep%edp&skV6&RIzHNTce|`G-^;I&JA!wIP`-HmTl%hkiuK2Rw7R
z1ZOUn4ED_Bk_4lXB6_kY(n^e3J9P5$QQs#oAAPHn7yZ&Fo$pZr-}2$W^l3lk#xf8(i!s8IRSJVNI(2HS+?xy-<}!<86qSM_JxcqU@7
zO<11J3~W1GiP84wNBSwZ@npnco3K2i8Q6BX5~Jt^@FQCV5VSf6M#yr0hBOrQwcG;DBwhtK#r46mc73GswmL
zzqxRGE0b+U^nhuHRHAoUhtz^ckY;(LPNq;#>5v>s@iw4MByHfh))-H6y5W?
zjbhPw2E6o<+xBc|7W+M=hi;3zy;sE26
zsw<`5yBha|d2r$7O}oDp<7yXfk*7k>3X6VYJWY(Ge+cm`E9{4E#ABG?E;z^INU;6w
zu@9!`fMpp@|Q??)TWXI_GRk6>cD(wCpt|i@3_N`gQz#=(EI)_)Qo-R
zwiTD-mDf>tBH=J@X6gl@+>Y#NGo0HA`$@H5A(QluU`mJBcW&=9={Y?~g|FQFkxY;L
zt19GjHizbyhI3}1F@tZykEM&2stb_);K%ed054Ua_H_JnE`c9Q)lX89G<>OAPY5ef
zGL|Zu6hkXjAhrro#)awzgoArv77Nui6n7JpeMGwtX?HmI13V|Q9*roS`VH>MrW0Su`<*WWd(E~$N#|MSS)i85^wV`~I0G-&|qKdepi<_&bw$^2
zV{PV5Sr1%nyDRsBOI~e7Aoqbu{^QXr@VZmXp6-`cz198VD!iV%BdN#OW+bks@(uB%
z(Q72|x3rCJIW*K3$cdxZNCZgIK~i8}OWSaJ%b_!K0y(35l=Mv5<#>OfD~0<1O4IfN
zKEM;DJ=TI^;+>2G*mpP6!JgUwvh<&6_BX@7F%Eo`+5fcUBh3ECr2lNQ{~`GEuUUvH
z5%1_A(@U;fY+knx7zxLER|~}^?+S#@^4`+joPSN|$yoE2_hPSG?7d!yBzOxUa*lUC
zXtdXkeVjKVzu+3mFba&g*z1-q_FhK9?%|&{&&)kEuNw`>6IL?4#R~8x*#gWW;$p?Obdy=XKo25Oz0qb`gZ%eZeMg(z-(W4l-%Sz%vaIb
zfzk9z4kXaq=)FAoFH}HIZKrDBR1KV}fm1bbss>Kgz^NKIRRgDL;Qx&pm{1m;k8{I=
z;eD!d(vcez+^){s!)5xVzy7;kFXriu~aPvGs6M
z;NA~63+`iZ7sEXOcO~2)+(x)@e@6b`PK4VGHx2FqxOTWl;4X$6hZ_OQ;f{s75^f6I
z#Vx33xD&Ud9pQHF=|7O%P*cX{(MTBgkKhZn>XT?2LWuv=nBTu3abR$XkeoeuE#GI{g>i2JX0Q(J2WxQ
z7Ozc2kj{@AAzwpr8Hq`EMGRfK!lkSYlPehe!
zm*V$6eg_cuAY5IKEphan(Y8d3KPIXMnjjBNIq`^2$8QA6Z+{B^GK5a?VfVkHG7?8O
zL}w&g?vBYyoOow!W@3te$ehHzQP&JjOaXx=NI()=?D)nuk}XEsdodpGho5vbMCB!p
zzB@XHZ001UL^+2hTC!xCEWNyUlRb>hdQNk9)I2J$Ju#)wSX|LIbfR1c*%>4&$_xD@
zYq?RB$A;2VJG4Ts5^_{Wv{8G12}X0l)dcV)c`anq;7vd~H-VQD?FN6FXmcKjd1~ITi`y=jt#L*a|d^$P*
z2{9O-;M9MKmoulie>eDfI#0Tu%XI~KD>$Y?u~AuRU3dpf&IM^;Wnc%u@b(M$t85xi
zqF>o)Jc)jFOL)(KDCuiL+9OCS>yFsdU_mexLu|x$gT1fFy#q%1Gp6kTqx>0J46Qv1
z8x8iF!Y&2dtFUyimlakBM)j4z`J?ur|ERkI43~OE-4>%YFziKtT8lU;4=%QeI2Y=d
z>g^$L4Quf|#KoAJ6RI~c4`F7&c)UriII0g?M+=eGSWa3~^A$!f*5(=*&G{q1>|p1U
zUC}qG#q~|}f5b0Fyli)hA)X6P`Yi;|f0lCpDPwt+{{Xxg@|0i5C(Yx2MQb_fr{dE_
z^+)SEm52T(5#`B`xsmcjX?Gxw@+;*<`N6UbY%%1HfTe@&!~Cb^kq!^mkL`ByMFr6y
z_V|1;NWR6MGGT!_I*!Tz@_V4__djVk(njAjrA>xcDB(Mm
zFn!~cHoW64wg)*Z*BEioi5M~7;1<^Ea%bs|Cck4UjyJ$)%+V%~iR5=nY5u1T!%=Ks
zozxG>igCO|n;hdS*)XP5v01RL(#EG1+Pj=Tc0%6>lWVrtB*ZmVtBvLeT1RNZ)Fif1
z>~6H^9?2WIU^1T;j=<@V*k}%-?Ia&kCb4`sho#*P4lm{e$2n}*lmEx}t-t8o(d$#iXweowl^82*%d$jWVv+{eh^82##d$RKTvGRMd^82vzd$6JJzh1BBPk!H3
ze$Q2Yzg2#(b!M?H_j7hnuq(f}dP9jGFTbZMzn?03x#{2r+M{-^xj
zr~JOB{GO-$ey99ir~E!AeN}-r`g#Iw@_U=|`TfN(aCwyrqT1GqDL0P
zRypi0gS#C1S0e2y=(-iUz7JhLK=?N3S`GglV1Br3!PbG@3I7k_--~?iLq7K-pKri@
z6Yc@nekVHm%)doPkN7+69tL|4?0wk(81~zvPakQ6n$@x5-57Fd05YA82TM7
zJPr?_JMpSdTnTlzJM3eDC)9
z4R~Mq-x7g0gHt{1oeB>&aSaphJNWgb=lux$vk3fH1b!j{KLZugm)-NgO@{sugY>jT
z(u04p+ozs;!Rbr9E2$w2+XKv-6#h%*e^K~vnIBg8ADAaCFerytJg5Y_;}!lq%cm=R
z8}m$s?_yr8@Hdz@D*Q0>tqKn^->2|TnIBd-T?3|i`%2+m%;WOI%ZpAWC61-G}twx^lX
ze^fJW;_D)ie?s9~S$>Pcw=;i9;k%i?sqnure_P@EnYSwZP39W@A)ax2hq*=JhnP=P
z_=n8%75)$Aw=4WObNWY!#;u$ABMOfd=kM{_9~BI`58sB
zq`JF=`B@5gFh57(F6Ngj{AT9!6@CkIhr(Ad_bB`}=CumvlRJrn6Fd#h0Nvk8L8(I=08^Cr!qgO@GF_i>pN0U2J`sqB+=b@
z%vUJ9kh#1*BvW0-e61p1%DhA2H#3*lm86~;=HDpttC&~NiGyLggSoseC8K`GyiSpC
zWPVuT4=|V4wWOY3GXG4Gf0X&srBYINH!_#k!({9qnJ-u5#aFb@pI=k>E|!;i6Vc5`A&s@#eA>AyO_VO@aR}QpOf&d2IH2%
zTwd3cal@ETSLDYqzf0j4FqhZ=q#k;%it6WQioBJ1yTWHNm)9qyo~xO6D)M&bxi?Bl
z-MxmnyuK-87csw1k#{hEQsHzToboTPk4imN%(p1=b<7hTQc`zUGndy@Wh~uGC;ewD
z@(s-03cruJye=#C{G9m@6!~8>->dK^nak_CQqS+1|4os9mbtZ9O6u+o<}($(hxt_s
z-_QKV3V)mV&lTRvyh-70%)eCl7tBv6{1|gni59+z8$$6~k-|@BF0W(DM2XBx75P!j
zH!0l0TwVv4dL}U6uEh28Y@;bVVy@L68MLwJPN`=p7F0aE&Jq66=^>@h^
zGvBS~DPcasDJ6CHCg$=wy^M7;zd(^+$=t2*HO%Gpe5vP7=65La_b`7=;rBCtSK+^4
z{=UK=Wu9LqE$Z%5%;o(G8M}#ju_E8h{AGo|$XwpQK
z@9#)GQ<%T3$Y04kZJCtR-8szV{UI5f%iONWFJk_*!i$;9`%6-fi}?;k-pxFP?>|ZT
zRm|mmDamhVp03ERXZ{<7|CqVFuO;=+zto_0$>qXLOx!rSMn*Z_x?=b^1AMRh0E)-qizn*
z=PuTtuke?d*D8D;^Nk8`W!|iCdd{2r)j@^ZqIDi4Z(MM;_%(E4KG4mS~ei`#Bg->OEyTWHOU#sw|nBSxDOy=r+b2;8$P~?k*
ze6rT8jF&R-zQ)V5tiMgs^AhvK>hOA#<0VDmds+TUh0FUR`3jfgt486kv!2@(evo;S
z!rPedQ23Y3UsL#T<_8silDT{yK$bdcDEZr0}K8
zpHg@+^XC=rV!m79w=n-w;dRU{HR0v@KJ!ZyUeEk;g|A~?tnj;-S1bG;=IZkOUy?b7Z#E)KH{ag&(G7#Mle1y9zNEB(+Of#))>
zQREAmuT=Q;%`8^c$MQqG;q@G3K1t!9FuzLSUogK~;c|VfQ}_v%_bc3VM&I+s5^$Pt8-E?1
zPnY2F+Rqg|ZsvFU!sTlt=>H4Lzo*FWXWpjpH<^E?@OPMhrSL<{zgD=sU)-(m4_SU%
zZFv6MnO~{!4(8bk|B88@!n?rxnxARj?29KR(sNK|Ep9`2ekjzJ{8-4-^Ii2eU0%Ec
z1n)#}T1S@Mq;t8SDtPbb2IsJz#;bM7SWa*g>zSwMSMds@DmbaK!De|8%SDzR9ig^v!k6uf7KZCh?{5f14fW&KG
zD#w`@GFP9AvocqY&#z$Kq|5_Zg7=>H=Ci!|yvKFmH0~_fdd}szD`R=}xtbc5Pf_~A
zkC->xbv?3QJ;HjDl=^&{<%^YZ`Udm-%<%ji2B-eKQR&ZLuzVBi5$`}E>1o6Ca&^oO
z*K;QGdd2P(=EX|6ikPP}m;HGaIE@#z-PbW!uS?tuZqj-`Z~P0^-#Ay#r>wV)LVlF?
zm{Q&r@Zqpq%m+Y2^kmve=9V-alrq0?ILY_kW<}ung5#i->qlHZMe>!BU!m)g$G=Z8
zubHBA(e2@VD}w$bEWdY_E-&+c?wNYITA81%M{C!BTi9=OiiNz9=zd=W-VlL5DtPbb
z)t?j`>(LZluPoQ=%r{=8bD96qBlP?yal1rw){2-nak(yKey89ZFgW)|;J=B$x3YeG
zCRJ=QKgT1;55p3LCg`<|7aaY0rXDq&C9Y?lujv0NxTTjCO?xVW{+A-~zeUh<4E+2)
z^q-lm*K-Q@!xWa8&Adrzms`B#pU?dq(}>ufi6H-1mQR|h%ONPXp`-NtG;u$v
zVV*5G2Mo>)5x9r-q$u_LICE`=uB4uIy(u}jmz#OF)WiL467x%Ophok?My|I!=1%5~
z%;k80NN^T1I8R03FR-3s)^i@G`HXop=O00_SE9#h>0tR3ZpR1N#-uTNeu}5-;C|*y1UF=ay&jfN;r?S`{xI`&rCe_@Z%oq#D_P!z
ziH_PuyIcpdeo~nyae41y`C{gctY4fKA?~Mwvyj30-w6D9)>E(K^GoJQO1Z|5)$>`;
z?Z}RH4fDO6PdR4&lF!sLb1moTx6GTa(1Dy+-)G*+`MHec&pk)CyYX@z%wax{xkbtU
zI_9mMK`w^&q~M&`;Iy#(M$RXiNo;RKkZ)&s3+LxJ-M#N$H_Ts7{>O2~Lz&bq29?=pDHD=J-HxNfm?
zyB%c}-kK7hbGcAiQSJ44ye^-k3}t{GstCuDCDv(5L|yjO3W#Z!IUFl4yQ!qEBL9SSgd4ls=Q}WRjgsvXe=6GRclqmz|o9
zf&kb=Aw?NQ5k(29fEj2-+6#**!eWZBm?A6!i<0UEMQN!LNKFQ*$wKC|$&=HkPnn!+
zm7Y|>X(BXLVt5436nL7%@CdA@oh~sv0;dY;84|-I@Dzd5B!)-eX#&rb7#@MA6P}U6
zm^=c{5ZEd)JOa-YI8|bJ1hxq*OF07`finb_rJNy4nI*6+Y
zhA3q=S8SThH+fPuU{SGYGT-D8SX6A9%r|)i78RQ&^GzOsMa8Dce3M6DQL$+<-{cWk
zRBW2eH+cjW6`LmWO&)nkVjxqu`^{U$s@3+*qO4F`Ylo
z@(3&{cBU*Pc?1>}J5!dDJgM4j>h0KxZaY(Eo;)H>)b31~dGZJ>YImm0Jb45bwF`yZ
zTZB(Ui2B8Q*`Wwg0GJgujL{v65Jff0_3gjYIRb#Av
zJ$Io>`A!cLaNN;T2M8GIfR>b8TJv(J=@k)*)+<5}VgaN>iwu&^vrfk^Rm;P~mxocC
zhl1ro#^!IDy#68
zm(bBKS9_7Gx=br@Cz18a14DbKuExW17x?SOe=HT>{^Ar
zPz#sDs^F&9xL5QRXD!H%$5pPmZ+38Y*iB`33cFJsT6JxeQ0+A+6Pa_Upwd;n%(q<2
zqJfr011&2Jy(WuBo2d8st`&JM-*R^uj`q^Ps5|9QPs$02ET}AFk+~E*mpa>AibaEQ
z#5@XqcY{-&BydCJ8gH3Y`b-?CV@I)55Q~!=kI&(*b9p=!=nMMEj$`&hTVA#UnRGaC>N8_`
ziFbCMZAzWC%oSQ9aGHkyk>bdpDAw*NB)>M}G%H5tS2L^E!^g0%mR9941mgLvv
z&#SA_P!kxoILql7$X4y@1&bDvJjSLVXi
z%1KZJ_32($M=FdWvo&Kb$*N-1H)t5IuoChc`Q^fO&HCzk0fj6G4A
z1vH+im~!FExK=B`$amocFr&hk+)m+4uR=PXBprwo
z_A_g03(@(sP)}zABGR#BI;wT?f_zb(Xd`j<)vGcQ5voklP0$|JPe1k8yy)w)LvR_wdkl2~
z76|o$P$x&0^g7ig&~%Qh>*m|?axy8aQ$lq?eKWi?WLTSv%MXqUw}Y=h;Q9<*fpFmJ
z$O^aTW=D1nu7TK`PM6o~_Bb$#v-20^<}R3@U0`2OI8fsGD7lxGJ3WKx>2q>(GBO>h
z_}@s=G;ei{2bW#S!)b)<;Npm5HeGw6>qlM}rsDpyty}ix)mHi{oanqX>wC&eoDs`Q
z1{UF(4b35FWlV2mXHa>QRYf+bf>yX|E6YkN-Oif_lS`+5&B<0)h6_;TZVc9CRqkqs
z7t5j#bEwxx>=;`7|TAGKEVae^DeWyN^o^A
z1K0Un9x4HP$%687bofZ7WL@>{R(0rggG@~JVWZGlGvA%%!m$jl(3O;h7r3gV5-kwU
z=sI3F8K4*S$|cR1TtH;Nizas0D2;yZ
z1EuSgJAG&t6>5&0!aQ7ql$TwxFwjL;9PQ#buH557W%bp2C?>pmgq}XNI2_KECDf^K
zjC-q+YJf`R)(Z{ILSN-TC8Z?j^q~$X)+-myOb#cS-siIRlT-UkN*9m4j;soFB3}ml1-OpvsKDjwEL^$uxL5sCX){Ic>ENi(zQl7w
zPabLB9H4sp?iD%n2TCWJ1szX}tHB7@xLkvp;4n}Ms>?7(45TzX=c0M9EvfjHQ;A5%
z)rzT{rnB(sql(5cb=XogK$Yq}wz{m)Jx~!yl7>+MjL>{I;5rpvX7qzhUnLS^6%hT@
zfvKQ`4&%OcevQLLpPzHZIZ^-9vvDP$S)3N{L7Aq65l2Xd+dbHxGSB6z!PTQWtni{G
z9V^fSdU7zp7{%~*)_Odcfazpueo3|4J5Y=DY5*~bW)JGnhSrmT^vfPWy@A&E0eb``
zz`Z7YW&M`f7i-UKakzy;#-3`UX~Wsaf;d3G?^QSGoOw8Bj*yHeSVIfsRX9BZHvyHH
zMkTFEgQ^}HVWG8Pz$zm3JdsAqd3B!$yP^Z4$los23Vg)Vl}!f$Xov_kCGE+QyL?4u
zH3JFvY7JpTEQNzGbCbKGS`25Oa?IG*4Ny9s>V5b$2rHO%dOEtO38fR#p$#qNbeI|*
zr5|_qFkCS#cUBu0=s_%gRH4w$>y5Yr{Ozih4kTscX_LLHs&IV?u8%H)!CfwW)o!0_
z^0MmM$)&Xwm1WC-0D
zjcaYP&$SYJy5EgBH(h?yCcBpN9q#32NGn5n3jJ2Moaq@X&sfHelp;u*OdJ7wnE`4`_Ho~MzrkBslq=S)enO^??T1N3Y9rv=|1a3@UbB8V{-G^j@b9yDHJr16TpX?v<
zy9mtPs=FbD|Dpo}6H8E}As6d}V^Qv(G%z^KGUZi*O}Oij_i@Hzz2&(nkg
zkF{mHiEfsg)soYNOt%3PymY}!-hYXvHRRS3ZLL|m=l6N$Jfy?+&#&Fz>o>30_q@N~
z&-0w;e4opC&N*oH9-+ELXxJR^>gLxUCj8e&V#03$-;HvX{(8Da2=H)&Ps?_XRGj@E%oeY`H0dFa(Auz^y@THJ>2bev|=cv`RXa-9Cf|hfi5?e
zfIX)WX%@?}T-G33bk8Z}HGq9uA#GFZgwcicoH}?4E08`)l>7|RDU;Myce1j`qV`0~
zxa>sx`3DF|k#5GF6s2$MKMipN(ghZ1=F{@LBnnS;Jn;?uR
zpvECVDl<(U66W?Q@`x*-j|{n6^(t*15`v>`$Y>ld4=EM`=F=qiG+{SRcMlMx0{WVJ
ztZ;t;z3e_(6IsAgJ1^EA(iijw6i_dZdBUxHTH=u`xO_X_>6oSo
zNg^Z@dFYd*4q12W7}ml0{cJW)U1ix6pOkg+$I&c?);-SV=oso~
z(2CaN8<&4>FIJR}i}fah+9>M|AER+zMI$XZNwg#x^x|Z7*UN-dZV)Z;%a74LUcpfa
z`!&(tn?dn8@$b{NGRH+-&LFw;i<;>6eodY_{bl;K*S+^TRCPRqCH~W+7^O38mFcW-
z%QSyI+ON^7yH=yL_27)P*FV|z{u$PSw<}Jd)4WE}4c^0oUPAH1C##=%mWWX&mL4sU
zyseP>HNI;465Z>aqY0bGuFdCCH;?-I#F=j$VfTpv744V3(P=hN7!zZMqu&;9Pn5{I
zOQ_{2>QaBV8#M+h_3p(vWw+;{*Ww&w8ZcTDakjC1)O6F$tn&+LxX5Rp0xBE1mQ!RLtn4_Y9jR4DY0m4ikmm!}Q>=`+fEt
zX7{DpuOdX&%|A?s4NvyT9*_*O&f_pG9e$s`zeB3>;)7Yu#!S}jJ46o*PZc^3QFp(k
z!l*;E(l1*O4$%+%(u9K@bd)Ykc({X3)Anyi)tmY)?sH}_ZpeuBGVOj@m
z)MW_0?ew}X+x%oZ#-(F&XNB6Hb5oY3bGYiHg2IkFrbZJ&vXyQwjQXJGqvlGJ6A
zKCHW_*;0;q7mJVY*Fv>#o6XiqC+-lUUVBT>JduYGD@giMl=|9w1+ndl
zo2dH6tO=>ciR4?&iVQ6dGxTDuEkUepPe71r!y&(~Omimv91h}By
zpiR@+zPU`ZQM8q5%8z6-uNyMzE`gQXa9^Lvy~*Sr91{A($!(R5(uLZqKS|wUo4LnT
zRM{t=sz?`$kC`m)NXyk!@9HyK65UM}(LGsmPpY4xN}AlUF$S^LRG;Ldt!)>0xj)J$
zpV~MaJC%=Kw1}_-qbQ}iOM+95L4`V7r6=0VRqkG-Fx4)9M?W4h
zRoMAF^&UAZ?Tv$&zmF5;)B81O70Z0NM9H^TG)J${(0mGA^jjdGjf%%X**5zA1@Sdr|bNaCT?(S-s;tWsXknO
z4i~b-ryoExxhJI_O`9}HJ*+vsSijz2MAMdWD-Y1`{I?tau^1ayiY6NSUS3};hV-Er^QPKrJOP>n}TuPY4Hb4w1;*0(rY?}sQr->^@!bqC>qL%0Tkbqm5?hj0o4>ji}E4xt?a
z>jebKA-n;BrP%v8LOua>o!(<8bR|toCVhYO4&loT8XKCb%Ao5)qj20E8fnh>#WW{E
zc1klGf>WA)5S-FA>e_gF{!F_nl>g1
z$I3C0=Ja38vJA3Qme~-TvP^>Dl;s2nPFaRQaLUpb!oOr0Xz&<1cOc6fbLqmc9YX3{
z+HFWx&7l{Ajlw5$XmWU@dCo89`3z*IJnJAh{II2fSnExAFF}vvotfrP7ZhcL*0!XhU#{
zur-ASh8u;_6sn7iFi%VQ#dISeI}NH1f>XM#5S-HeRveV>RR~V$o`7&ix@@PVPe$E8
zd@mlscWk?I_^lLrE$R=#f)uM19jp?ji`G4N1qwpwEbHNMFRFspB@yzNkNEfVrh4^5
zGx6;r8*EB+ex|i<{12+2vKhB)oJ_d0#-T*(_Bby;-NYF7ggKX(yRa}nuR!j>GxV+)
z`d)Hij1NjWMCk|~G?jSJsQ)<*Px3B2<>B|k??Bb7j16-5m*cFiv$neU=0)Qgx%qjC
z@|7~kSEK2RDYJxK(ey7Ve-ai&(ch;&CR9by5oyiB86#~?J8n`L|K?Kvl_TPg
z+HYa)ov>G`TM0?P3;#tF-I>#I#P*{~PVYz3k4=|^-U#~I>;U1z2@K9>#@XoVRL}hOTm+
zC87SgOtd{D&Fote=dJGZbF;v39Bs=UrC)-ImK=8oxme&POOn68
zeLA*35r4j@_cn~9x3XgdzXddUo?iHRJ~hn?4_nIaBTH}v(53#f<8>QC)FBaXjQ9js
zTjx_cFHMMvp*{0#LdjUVE2nzQX#AC;?xbdm3v1gFpSA%T?M1PE!%vQ{SaeQRK$COB
zg){kdS*}g!xr@f+Mfq&NO=j=Dg;VOiCr8uLypZsl>}6Hzm34F1Inp@7-eK!5R!lNk
z!as0G)3VpmKjuvjNJivnoqtu~m2;UvZ&8=#QC0r58Pjp1GzqW#vW~80O{u+DQ_?r0
z6f8KrnHaYtD7vw$_K19AM?<#b8&uDm`8=0a=LZhElIu7L6X0mX`;A=sTK>JkNr?F0
zE(#ZP-2VQ?_~hkMbwQ+d5>8~_Z@AwvjuE+ZYC+%xZ--d%ncWNHbmP+P^DOFq$9rkz
zatpp7@quzvRe2Dza5{%RSr8W74N2@$?*`bfOZ^A#$vW>f^k_k}y97bjjafr~EU*fN
zM%pr;3P&Sp@`8;*N+dnJV7lNDN!5iL1u25o6;2mYBIx15EyBCu*6f7`1#``#{4V_e
zp*j4Q6S%!ioLr?O*v`3xGlSEU^VUjM4|lN9J`&$=`0}v5Bg?fnJ;i$U6P*w=$?ce=
zG`vdLzefowTz*G3^BTKv*9c=ZK2d7C{&N!*w=4WG2Xn%|Qt@^b?$^NlI+i^9DP(Ff~~zc#Cr&fz10A{Y_4tsKl|e6t3p+
zSpAyw
zKF_(C5%2zF4Hx!sKFe9hxten~=hK`uoR4y9^U)wpbeUjnDATHhs+DxtLos3L%bnA$
zUZKRzT(4eE`ybk(E?=aa931$9W68P`g}BsN6QFyn9fb?tB9gv;Mcv4Gxa08=T7tagMdTTLT?I>!?TVdF7y
zHt0Sm)((pGeb!Y=f9q;)!M_G+3AyZxcWd}P;MC!S1i?=LGhngsi@+7Id*GLY>tU1N
zSA&nkrowLkcfk_iw}9=iB>3In6_^OW58Q}Piuv%XLGyJS7PA7-M@L8nd@c9_Y(4yD
z@DOY>{BH0ptO0%xs2PEO9AfpsaMwzmB8-U8sImB{jg`@lTcg`)(oG~3flwU26~2JJ;2w38L(DX2Mjl0{tqFDh4TvR
zH2fCuZP+FFHn0QM1HT&_5{~JCuLajf5b_uJ)nF68UR?L#;^0Vp2l>O-gOM-;{8+Fa
zW`y4WJ`Iay^^KVSw-DTezy=$IHj;nC@2T(V4GPb@M%~B
z{AO@BY#aRaB0RBS&oUhhScKIMUk}~~dlP;FSPpB2&)5$;1fTr_vkUArd@Z=fjKetu
z)!@})tZ?{!;OvL6lHsR=$6;D4Mg^R^7()s_9qfhO1HTWv`Y=X=`C!lzl*8A9<*+RH
zj3bs3lFM|k8MYX{nK@@+%MkQ{nafZaei8T#tO9;B81)ENKm1tmYuI}Dec-pSYWNAu
z37G}k1V0^I4ciRA8vH$M8~kRlY6V7)>EPEZNF;d`fvud7u`4mJEw}(U1(pRr0ZfJ+
zu+q>K7xt|_tnzkO6Zdx4SeGK9T$OHZf9!4{Y~Ww=KRBWMjUyW4%*@!0F@sf|pb=~D
zr3==M@nox;t?F0=|^E!>b!OnC}@v(qMJ4{jjUBVQ2AY1z0Jp;Ve4e2Iq6w&~t>$ge`+@
zgB^f<1`9fme;hkcm#@oGMOoX{wF*O*mt?KTEM8PmQM!t%Hbz+2Y&5AKtm;8``7fat
z_sLa^-IW0a8#A=3&oIGvXa}JAU&G2m-GSdZXd<*pcW6OhVnJNP|0gipQNn)TbNXCY
z;t0+uoavk;oE4m#Ik$7Z$=S{MC8zMY(rysvB!y;@&V>@rWt>%EFPB~CkMALlJj;R|*qwhAwYgT01xG^dgCZqBKkvpCZ^^Epd6mvgS=97wQ+$1ia1
zW5o5zF)r9S2TpLcE1s4!h;uCGK)oqEHgOhmF5_Iyxry^B&TSyOKH0^ES2*9~Jiyt>
zd75+J!vBG_cNWjTvv%sq%w;Pc
zEwz5UIn-rn2Fb}x%q(TwApNT)&MnjtQ@^L`F+V-H%>SXKC1=b@HchxYreqnOh}Z^j
zt=op{-wyuw{v(gBE?!lkG;Kt~gDtbPxN;T0_EePLX*rp9FPxCd^GZkIud$rWxiiz0
z3s@7k^sDHoEhD`hpMeBV6ZSa>+2+z1>*`&u{HL+pokcrIHT7wFUe!QfX}VX{XZ^e>
GK=nVnP@?q!

delta 9445
zcmc(k3sh89+sF4D7;XYHD5G3uM6O;CM9e@C6mWz|kw8FDChylo1tT*DL(6HxMz)#S
z1yb|V>`P-%(e!Nu6)$CIYF@gaZxmz}YQ1W2U+4YrIeXF}*SBh|ul3DZ|M@-7fA6!;
zbN1Qi%sEi8Td3GBm{$AgH!Pk__@|RZ1$PsKNTP#bTDy6Wn^H!IBO%_q$=WMX11yB(
zyP~OukXA<9$3#eg+?FuP$WLqg_{(c62#HwRr-xJRSp>q`K7QS!N~f2XqDO5pzLb!T
zZj$wNEkT6+kZJUoEXcJ2kqb17{vh)kY8@K8AWjhL_`oNbkmRWHX-||sLC9_gG}Jp6~|_6tEN#77o2Jc=~!1p-5mRcAY5fxKDn}8rZEOySwu)$%-0{HGcm}IqD^ZY
zc)F41JBIW`lhL{IuAo?X46XZv@YTL@60LTOR-j2^EOAMTxuG@2hU
z0Sj3R(X6tRdOT)mJ;18d819KJmGX0M(iC|Qr#CT5Y7w0%j~{65>(K}_JFx=VpDIF`
zQ$RN=fc8m6^srndBo)z%^1yi3U^ty(@D4G2sx>sPI$NeQ%H1$Bc008F
z-Z(FwN5%AQ%i3<#R@|UXoPcs
zYynMk4t9DSb&M~d^PLCFUZ=IrLHM>f_r~{E&Qpc>i8R$EQ#gvvUHpU{1@ymNqJ^0S
z^edO%iqQqEv`d=GBkIDA_yX$gIzjL&pwnE_gxd${PS=Qtj)R2EPz0^S@uF+x_vNDM
zG^~^c4a*IOHB~tw>h}H``ft|&=S_r&D)k}S+ijRDR=?J8to=?NEp`hOqDRx^ZvI05
z(eyR9c;QYu{md=URhNe$j4H3$^ncu{h0t`mwp)OHCPwdwWRn_Y+ikl;Iy@Eg6hWzk
zJdQflsfH#|_1yuigG&e4+4%Gtt7ht1Q5E_DhQ-vn#8|8chJGi}&*(9219~@#sv+pm
zfYHS3U;RE_w8n5sQ>6`&M~JFD2WX7DuGcfzNMlS5QEMKRH@`|q*|QpB+~fmvyL(_H
z(p!pt9;-pkM$MhXms#K-FJUk^+|nreHQrJb$g^IhKe#6(SY3G>gE8*(e$3JZw#rmn
z5YABic;c2qDQ}*E)`mkPm$sc>_u&OLf`=nEpo#9j^jVMY0qfB``)tMf4MY=pX6}h;
z1`ji=TM92ZT~Bv<1RdKI?l@>RQJ9m6ChKt1YD
zsy;$5Yg4oQ=U33f)~Vx4H`I6kJal|upW`%Ow!*Nr(Zhlb4UKGUjTmZ#WipvG^sr}$
z@WOujt!Ij`azE{-cuSbwKtEO_2|F67hcZkU-$3J(A%a5#EmS7<**YF$+|wAE8rXF>
z!{8vQPV6H@vG25Dl%uFRx{vNx2Fs?>&y|TiE74br^A&+hM`O5!0~b}j_R&FJX+ldD
zUE-zgxoxj?gmBA8N#-D
z>f$|DxM!wi-nqgxGyTXrQ+UHneO1B27&A>&`S>}TZA-9?)s}_ToN72Bs+`Sqfl4g|
zn(10qws4}3wyARc=hR_dnjSq?DsRubD~g7T*z2&u$xWDI8tU_e@0ahhV#?fL=SyeS
zkQy?){gXSi#%H*&u!?@+lb*S77Y6wxMLc&vfm(4NY=D7FCPlo!OzF;dSYwoLu=-f`
zVhdL5CS?2BT^&mn)15D>N_NpFeR~U056(
zTEnUKO85o`bW;~LJ$Pv_|G|7Na*Sj9JZ_Y`j$vo%=^@SXe{0M?1}J6v4)kGD+rYjs
zoT*uU0T#PRt(RkAQ?vXi40dj;mjpBFVX%jS2KL1aRw?@lZ(v`{sD{CQvm4kKGgiQG
zWCr_UMhOfTk^+Z44lt(}jw|cJzL+rvG5!Woo&HTgOHNuPO
zR2MWx7?DnQ21N*}bb3B0On)OyI<$@(zloylK90kK%?>Zd((y4GaCk7-;lVg)HR@r=
zC1a=6*aCwc2zqR?8r3k^fxs|WjTJChl^rXPO6Nd@6!VXAXQNRnw=j*qtgaCbq@vsm
zSt_cH#P|82aDD1SYjuKcSL@HCy43nL3_13LN~Bspg<)4~BMiG*--pqqRzjaqyLN*w
zm1!JBi-T)~??%$-J{hu+^qD@9_ZuFOyvc2nvN!>&vR7cq6x7PmySprqn-)eiB|IEKQ`f
zVHvVS6dQ@}wumtO7sDS~>@nDO#qNP&SL{|8cEwT{cEvsi!>-tcFuD{=4}`mQgDsVr
zIh@u+)(Gz;(2|G@SpxkaA`;(i5#joThgSKo_%2np!LY0H91OcEn_$>gSqH-?i9df_N=D`p4SQl%a6YK7+a#G`m!*DSrzj*S*
zf4B4$e&n{B3_*VyDos1W~K{^0_efaT|$gMov(dL(D>7Dw6(%EKf3y{Q`$4Wf7t&q
zW?`p2V@|aHf`xXby-bdWvSd6B?nKjrHNA$s{e)E0%f7T`?B~LMANuS#Kj94@^Tu%-
zWkNq6I%NDWLR2proqbaO;!|vf#n7dkb2uk*W^l%H_T%*9bmRPGi8Ri4oR>LIa@KKf
z=B&0-q37=+V(22?FoiRlb0p_rD{VhMV(64IX@}XIBRL0i26J}j6gY2{O50ttG8apV
z8+DwUIjcFJ<6Oi!g)^ITu$8$QLmPXM!D{KFqLXrbyqr}yX;oP+8e_5x9!u_e)8}&{
zg)jQj_j6K&jC%T0PE_|zc<#kxi<9)6*DQa2JROpYr^{fvBDbIGiV8MD+|mkKpZlt?
zP)#43Fv`0mi!F~+tNx`}=RsIutI2?M;o9mh7xZP6ICK0cz#Wf6_a
zQw#5pqgioE@TCi?G;tti14)aN)UZ3&sUv+j>bF}7{joj
zwZ>E*pY|l&?KF6ot@3aTqL=fcgrCMzw|up5dMq7~9~zv=t{Y=u4$v%rb1arAKAOlQ
z8ayK6eN0Zr(#83if~T5V@-0GYZ~9z8Mc?1F*#B9@8V5GEG0uMh?yFZcRnPuveWk&{
zUDeXiiJ`(?E!9r62$n$VSs3X#2^Up+%P-iZ%ELE+rWFQ-USvPqP_tugdH6pQ
zw=>GmTARwAG|G=7s=O;JJA|`PKZeek6dbr4mZn+03SggRc_oiT)faQfTjlIlqPaG2T
z+0*$|`2RzF=)()Rf1B7mO)9XOGoCY;^YT=w{iK!b9k@Lle@(!vM$0}?ti0={sgkch
z21Ak7h?CSm1uM@<>hR+}QO|qqx!*%zJ=(A>hWN+tL#({t;Zicr34bv$S$FsqPAY%3
z=;*ff_n2Ai?;7mW`s5o)>^D@mKq5Pv_|OH@7go>k((kr&3EM*IAbpz>r1rWbiTy`N
z+{#&-DCL3464!GoQl$JYZ(q%G*+|J>%Bj!d1K;MxyirnzHcndsm1&ayHD_A7l7K;CX+I)c%r0JqhLm-rx;m$4DJ^aVj&Ud_ktf>zp&SQl7)x|H<>>
z$0Yv>=Pur!H&*gLoToS&81a)pc5ve&=W)(GoZC3ha~|ca
z%(E}IcD|HbxL-Su-kQBeZYq{G4hZO!+Fdx^-9{Mkp0=3wRaMAnpX0dlOQGxB}F-;cFIa06p=bQ;Jv#z6mWu
zTniq7RwHf!KZ8t&TR}xnLe{bVU?}uD;%G1y+KMLq}?H
z!Brrr^CjdXG+4l|p^Jz+!5)6t5!(Q)fZ7o=u7Wxdo4{9~pAgrAhoHNNEufP>A>=I-
z2WCSqh;`t%kP>kxI4uAdybqERJclfSLJ(JgC!q+$E#QPeOgmy7SPmt#KHw%O6>%-t
zt2d?>u^P;Va#(+`8Y*Iaz-!Pn#GT*}Jd4dkoUBLi5>$%B1YUraA#MfxsR>z$I2s%Z
zy~H*E*Fh%4wcssi17gw#JA`TxGn%38h%KO7FxCTNC72C0Al8AGA^i~~tq8t>K0@3H
z-iBHblfHyJ0-Zyg489ve$mfW6gN>n>cEl~<=rBz0WSYS;tIwnJS!s30==U#
z?TFRjH!+xY#GT;YSWG)&3;28-rX8^fd<81qtjClg*aMZbPT*0f0&xp?9$JaGWHP1{
zTFv@^@x?d|h?Bu$Xe(kJXo9vQX6&vbq#m&v>~K7qIu)Gi?YL|cm;;B8?dl!I6YPKVyzLSszd)RiBVna@8xNT68`
z%3#}nZyOIwf4hjpBr!!;uy~=8!!&GJ0-e4r$gMS4TFbg%x@Or({TXcG+KBmrWGd2Tf5ZcaE3|{mD-Z^;r=kOn6{c*Ig&jrpOIGxT*
zz8_~aXDVk7=Pb^poa;DubDre9#`!C!&jsuFdJ-)KB$ZRkna??evxIXwr-@T46Vt=<
z_c#x79_MV~yv%vcNuI7y59LAZ>IhJz*=QPfFAlpA#!j0vet2x(m+IIUl
ztQG72$)8*-f5Tdl_D}xYV)+}^ik_d_KebqJ9{#V_O2%JVEdTpjF<*Xhr~?gH6HOV)!z|xbv5S;Y
V&-Jg#D(E}w6J)LCZ`S+C{sYkCiAVqd


From 174cbac8bdd467bfdc97428e2373e9b6f4a42128 Mon Sep 17 00:00:00 2001
From: Jaromir Hamala 
Date: Mon, 9 Mar 2026 10:01:22 +0100
Subject: [PATCH 171/230] test client support udp

---
 .../line/tcp/v4/QwpAllocationTestClient.java  | 86 ++++++++++++++-----
 1 file changed, 63 insertions(+), 23 deletions(-)

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
index 46c0a9c..7d0dc76 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
@@ -33,11 +33,12 @@
 /**
  * Test client for ILP allocation profiling.
  * 
      
- * Supports 3 protocol modes:
+ * Supports 4 protocol modes:
  * 
        
  *   
      • ilp-tcp: Old ILP text protocol over TCP (port 9009)
        • 
  *   
      • ilp-http: Old ILP text protocol over HTTP (port 9000)
        • 
  *   
      • qwp-websocket: New QWP binary protocol over WebSocket (port 9000)
        • 
+ *   
      • qwp-udp: New QWP binary protocol over UDP (port 9007)
        • 
  * 
      
  * 
      
  * Sends rows with various column types to exercise all code paths.
@@ -48,18 +49,21 @@
  * java -cp ... QwpAllocationTestClient [options]
  *
  * Options:
- *   --protocol=PROTOCOL   Protocol: ilp-tcp, ilp-http, qwp-websocket (default: qwp-websocket)
- *   --host=HOST           Server host (default: localhost)
- *   --port=PORT           Server port (default: 9009 for TCP, 9000 for HTTP)
- *   --rows=N              Total rows to send (default: 10000000)
- *   --batch=N             Batch/flush size (default: 10000)
- *   --warmup=N            Warmup rows (default: 100000)
- *   --report=N            Report progress every N rows (default: 1000000)
- *   --no-warmup           Skip warmup phase
- *   --help                Show this help
+ *   --protocol=PROTOCOL       Protocol: ilp-tcp, ilp-http, qwp-websocket, qwp-udp (default: qwp-websocket)
+ *   --host=HOST               Server host (default: localhost)
+ *   --port=PORT               Server port (default: 9009 for TCP, 9000 for HTTP/WS, 9007 for UDP)
+ *   --rows=N                  Total rows to send (default: 10000000)
+ *   --batch=N                 Batch/flush size (default: 10000)
+ *   --max-datagram-size=N     Max datagram size in bytes (UDP only, default: 1400)
+ *   --warmup=N                Warmup rows (default: 100000)
+ *   --report=N                Report progress every N rows (default: 1000000)
+ *   --target-throughput=N      Target throughput in rows/sec (0 = unlimited, default: 0)
+ *   --no-warmup               Skip warmup phase
+ *   --help                    Show this help
  *
  * Examples:
  *   QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000
+ *   QwpAllocationTestClient --protocol=qwp-udp --rows=1000000 --max-datagram-size=8192
  *   QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server --port=9009
  *
      */ @@ -71,12 +75,15 @@ public class QwpAllocationTestClient { // Default configuration private static final String DEFAULT_HOST = "localhost"; private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8) + private static final int DEFAULT_MAX_DATAGRAM_SIZE = 0; // 0 = use protocol default (1400) private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; private static final int DEFAULT_ROWS = 80_000_000; + private static final int DEFAULT_TARGET_THROUGHPUT = 0; // 0 = unlimited private static final int DEFAULT_WARMUP_ROWS = 100_000; private static final String PROTOCOL_ILP_HTTP = "ilp-http"; // Protocol modes private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; + private static final String PROTOCOL_QWP_UDP = "qwp-udp"; private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; private static final String[] STRINGS = { "New York", "London", "Tokyo", "Paris", "Berlin", "Sydney", "Toronto", "Singapore", @@ -99,8 +106,10 @@ public static void main(String[] args) { int flushBytes = DEFAULT_FLUSH_BYTES; long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS; int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW; + int maxDatagramSize = DEFAULT_MAX_DATAGRAM_SIZE; int warmupRows = DEFAULT_WARMUP_ROWS; int reportInterval = DEFAULT_REPORT_INTERVAL; + int targetThroughput = DEFAULT_TARGET_THROUGHPUT; for (String arg : args) { if (arg.equals("--help") || arg.equals("-h")) { @@ -122,10 +131,14 @@ public static void main(String[] args) { flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length())); } else if (arg.startsWith("--in-flight-window=")) { inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length())); + } else if (arg.startsWith("--max-datagram-size=")) { + maxDatagramSize = Integer.parseInt(arg.substring("--max-datagram-size=".length())); } else if (arg.startsWith("--warmup=")) { warmupRows = Integer.parseInt(arg.substring("--warmup=".length())); } else if (arg.startsWith("--report=")) { reportInterval = Integer.parseInt(arg.substring("--report=".length())); + } else if (arg.startsWith("--target-throughput=")) { + targetThroughput = Integer.parseInt(arg.substring("--target-throughput=".length())); } else if (arg.equals("--no-warmup")) { warmupRows = 0; } else if (!arg.startsWith("--")) { @@ -153,13 +166,15 @@ public static void main(String[] args) { System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes))); System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms")); System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow)); + System.out.println("Max datagram size: " + (maxDatagramSize == 0 ? "(default: 1400)" : maxDatagramSize)); System.out.println("Warmup rows: " + String.format("%,d", warmupRows)); System.out.println("Report interval: " + String.format("%,d", reportInterval)); + System.out.println("Target throughput: " + (targetThroughput == 0 ? "(unlimited)" : String.format("%,d", targetThroughput) + " rows/sec")); System.out.println(); try { runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, warmupRows, reportInterval); + inFlightWindow, maxDatagramSize, warmupRows, reportInterval, targetThroughput); } catch (Exception e) { System.err.println("Error: " + e.getMessage()); e.printStackTrace(System.err); @@ -169,7 +184,7 @@ public static void main(String[] args) { private static Sender createSender(String protocol, String host, int port, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow) { + int inFlightWindow, int maxDatagramSize) { switch (protocol) { case PROTOCOL_ILP_TCP: return Sender.builder(Sender.Transport.TCP) @@ -183,18 +198,24 @@ private static Sender createSender(String protocol, String host, int port, .autoFlushRows(batchSize) .build(); case PROTOCOL_QWP_WEBSOCKET: - Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) + Sender.LineSenderBuilder wsBuilder = Sender.builder(Sender.Transport.WEBSOCKET) .address(host) .port(port) .asyncMode(true); - if (batchSize > 0) b.autoFlushRows(batchSize); - if (flushBytes > 0) b.autoFlushBytes(flushBytes); - if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); - if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - return b.build(); + if (batchSize > 0) wsBuilder.autoFlushRows(batchSize); + if (flushBytes > 0) wsBuilder.autoFlushBytes(flushBytes); + if (flushIntervalMs > 0) wsBuilder.autoFlushIntervalMillis((int) flushIntervalMs); + if (inFlightWindow > 0) wsBuilder.inFlightWindowSize(inFlightWindow); + return wsBuilder.build(); + case PROTOCOL_QWP_UDP: + Sender.LineSenderBuilder udpBuilder = Sender.builder(Sender.Transport.UDP) + .address(host) + .port(port); + if (maxDatagramSize > 0) udpBuilder.maxDatagramSize(maxDatagramSize); + return udpBuilder.build(); default: throw new IllegalArgumentException("Unknown protocol: " + protocol + - ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); + ". Use one of: ilp-tcp, ilp-http, qwp-websocket, qwp-udp"); } } @@ -218,6 +239,9 @@ private static int getDefaultPort(String protocol) { if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { return 9000; } + if (PROTOCOL_QWP_UDP.equals(protocol)) { + return 9007; + } return 9009; } @@ -229,15 +253,17 @@ private static void printUsage() { System.out.println("Options:"); System.out.println(" --protocol=PROTOCOL Protocol to use (default: qwp-websocket)"); System.out.println(" --host=HOST Server host (default: localhost)"); - System.out.println(" --port=PORT Server port (default: 9009 for TCP, 9000 for HTTP/WebSocket)"); + System.out.println(" --port=PORT Server port (default: 9009 for TCP, 9000 for HTTP/WS, 9007 for UDP)"); System.out.println(" --rows=N Total rows to send (default: 80000000)"); System.out.println(" --batch=N Auto-flush after N rows (default: 10000)"); System.out.println(" --flush-bytes=N Auto-flush after N bytes (default: protocol default)"); System.out.println(" --flush-interval-ms=N Auto-flush after N ms (default: protocol default)"); System.out.println(" --in-flight-window=N Max batches awaiting server ACK (default: 8, WebSocket only)"); System.out.println(" --send-queue=N Max batches waiting to send (default: 16, WebSocket only)"); + System.out.println(" --max-datagram-size=N Max datagram size in bytes (default: 1400, UDP only)"); System.out.println(" --warmup=N Warmup rows (default: 100000)"); System.out.println(" --report=N Report progress every N rows (default: 1000000)"); + System.out.println(" --target-throughput=N Target throughput in rows/sec (0 = unlimited, default: 0)"); System.out.println(" --no-warmup Skip warmup phase"); System.out.println(" --help Show this help"); System.out.println(); @@ -245,21 +271,24 @@ private static void printUsage() { System.out.println(" ilp-tcp Old ILP text protocol over TCP (default port: 9009)"); System.out.println(" ilp-http Old ILP text protocol over HTTP (default port: 9000)"); System.out.println(" qwp-websocket New QWP binary protocol over WebSocket (default port: 9000)"); + System.out.println(" qwp-udp New QWP binary protocol over UDP (default port: 9007)"); System.out.println(); System.out.println("Examples:"); System.out.println(" QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000"); + System.out.println(" QwpAllocationTestClient --protocol=qwp-udp --rows=1000000 --max-datagram-size=8192"); System.out.println(" QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server"); System.out.println(" QwpAllocationTestClient --protocol=ilp-tcp --rows=100000 --no-warmup"); } private static void runTest(String protocol, String host, int port, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, - int warmupRows, int reportInterval) throws IOException { + int inFlightWindow, int maxDatagramSize, + int warmupRows, int reportInterval, + int targetThroughput) throws IOException { System.out.println("Connecting to " + host + ":" + port + "..."); try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, - inFlightWindow)) { + inFlightWindow, maxDatagramSize)) { System.out.println("Connected! Protocol: " + protocol); System.out.println(); @@ -290,10 +319,21 @@ private static void runTest(String protocol, String host, int port, int totalRow long startTime = System.nanoTime(); long lastReportTime = startTime; int lastReportRows = 0; + // Pacing: check every ~0.1ms worth of rows to keep bursts small + int paceCheckInterval = targetThroughput > 0 ? Math.max(1, targetThroughput / 10_000) : 0; + double nanosPerRow = targetThroughput > 0 ? 1_000_000_000.0 / targetThroughput : 0; for (int i = 0; i < totalRows; i++) { sendRow(sender, i); + // Pacing: busy-spin until we're back on schedule + if (nanosPerRow > 0 && (i + 1) % paceCheckInterval == 0) { + long expectedElapsedNanos = (long) ((i + 1) * nanosPerRow); + while (System.nanoTime() - startTime < expectedElapsedNanos) { + Thread.onSpinWait(); + } + } + // Report progress if (reportInterval > 0 && (i + 1) % reportInterval == 0) { long now = System.nanoTime(); From 5084b8cdb8566b27015ca08c33ae4946321093bd Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 9 Mar 2026 11:20:20 +0100 Subject: [PATCH 172/230] optimization --- .../cutlass/qwp/client/QwpUdpSender.java | 264 +++++++++--------- .../cutlass/qwp/client/QwpUdpSenderTest.java | 132 +++++++++ 2 files changed, 271 insertions(+), 125 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index c2399e4..37b7638 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -64,6 +64,7 @@ * the row back into column storage. */ public class QwpUdpSender implements Sender { + private static final int ADAPTIVE_HEADROOM_EWMA_SHIFT = 2; private static final int VARINT_INT_UPPER_BOUND = 5; private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); @@ -74,6 +75,7 @@ public class QwpUdpSender implements Sender { private final SegmentedNativeBufferWriter payloadWriter = new SegmentedNativeBufferWriter(); private final boolean trackDatagramEstimate; private final NativeSegmentList datagramSegments = new NativeSegmentList(); + private final CharSequenceObjHashMap tableHeadroomStates; private final CharSequenceObjHashMap tableBuffers; private InProgressColumnState[] inProgressColumns = new InProgressColumnState[8]; @@ -82,6 +84,7 @@ public class QwpUdpSender implements Sender { private boolean closed; private long committedDatagramEstimate; private int inProgressRowValueCount; + private TableHeadroomState currentTableHeadroomState; private QwpTableBuffer currentTableBuffer; private String currentTableName; @@ -95,15 +98,10 @@ public class QwpUdpSender implements Sender { private int[] prefixSizeBefore = new int[8]; private int[] prefixValueCountBefore = new int[8]; - // columns that need NULL/default fill for the current row (columns not yet written to) - private QwpTableBuffer.ColumnBuffer[] rowFillColumns = new QwpTableBuffer.ColumnBuffer[8]; - // maps column index -> 1-based position in rowFillColumns (0 means absent) - private int[] rowFillColumnPositions = new int[8]; // per-column marks to detect duplicate writes within a single row; compared against currentRowMark private int[] stagedColumnMarks = new int[8]; // monotonically increasing mark; incremented per row to invalidate stagedColumnMarks without clearing private int currentRowMark = 1; - private int rowFillColumnCount; private int inProgressColumnCount; public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { @@ -112,6 +110,7 @@ public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl, int maxDatagramSize) { this.channel = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); + this.tableHeadroomStates = new CharSequenceObjHashMap<>(); this.tableBuffers = new CharSequenceObjHashMap<>(); this.maxDatagramSize = maxDatagramSize; this.trackDatagramEstimate = maxDatagramSize > 0; @@ -196,6 +195,8 @@ public void close() { } } tableBuffers.clear(); + tableHeadroomStates.clear(); + currentTableHeadroomState = null; channel.close(); payloadWriter.close(); datagramSegments.close(); @@ -424,9 +425,11 @@ public void reset() { } } currentTableBuffer = null; + currentTableHeadroomState = null; currentTableName = null; clearTransientRowState(); resetCommittedDatagramEstimate(); + tableHeadroomStates.clear(); } @Override @@ -475,7 +478,11 @@ public Sender table(CharSequence tableName) { currentTableBuffer = new QwpTableBuffer(currentTableName); tableBuffers.put(currentTableName, currentTableBuffer); } - rebuildPendingFillColumnsForCurrentTable(); + currentTableHeadroomState = tableHeadroomStates.get(currentTableName); + if (currentTableHeadroomState == null) { + currentTableHeadroomState = new TableHeadroomState(); + tableHeadroomStates.put(currentTableName, currentTableHeadroomState); + } return this; } @@ -534,7 +541,6 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence columnName) { int columnIndex = column.getIndex(); ensureStagedColumnMarkCapacity(columnIndex + 1); - ensureRowFillPositionCapacity(columnIndex + 1); if (stagedColumnMarks[columnIndex] == currentRowMark) { if (columnName != null && columnName.isEmpty()) { throw new LineSenderException("designated timestamp already set for current row"); @@ -543,7 +549,6 @@ private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence c } stagedColumnMarks[columnIndex] = currentRowMark; appendInProgressColumnState(column); - removePendingFillColumn(column); } private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) { @@ -557,10 +562,15 @@ private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) { inProgressColumnCount++; } + private void completeColumnWrite() { + inProgressColumns[inProgressColumnCount - 1].captureAfterWrite(); + } + private void stageBooleanColumnValue(CharSequence name, boolean value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); beginColumnWrite(col, name); col.addBoolean(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -568,6 +578,7 @@ private void stageDecimal128ColumnValue(CharSequence name, Decimal128 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); beginColumnWrite(col, name); col.addDecimal128(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -575,6 +586,7 @@ private void stageDecimal256ColumnValue(CharSequence name, Decimal256 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); beginColumnWrite(col, name); col.addDecimal256(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -582,6 +594,7 @@ private void stageDecimal64ColumnValue(CharSequence name, Decimal64 value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); beginColumnWrite(col, name); col.addDecimal64(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -600,12 +613,14 @@ private void stageDesignatedTimestampValue(long value, boolean nanos) { } beginColumnWrite(col, ""); col.addLong(value); + completeColumnWrite(); } private void stageDoubleArrayColumnValue(CharSequence name, Object value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); beginColumnWrite(col, name); appendDoubleArrayValue(col, value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -613,6 +628,7 @@ private void stageDoubleColumnValue(CharSequence name, double value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); beginColumnWrite(col, name); col.addDouble(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -620,6 +636,7 @@ private void stageLongArrayColumnValue(CharSequence name, Object value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); beginColumnWrite(col, name); appendLongArrayValue(col, value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -627,6 +644,7 @@ private void stageLongColumnValue(CharSequence name, long value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); beginColumnWrite(col, name); col.addLong(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -634,6 +652,7 @@ private void stageStringColumnValue(CharSequence name, CharSequence value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); beginColumnWrite(col, name); col.addString(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -641,6 +660,7 @@ private void stageSymbolColumnValue(CharSequence name, CharSequence value) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); beginColumnWrite(col, name); col.addSymbol(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -648,6 +668,7 @@ private void stageTimestampColumnValue(CharSequence name, byte type, long value) QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); beginColumnWrite(col, name); col.addLong(value); + completeColumnWrite(); inProgressRowValueCount++; } @@ -697,6 +718,7 @@ private void stageNullArrayColumnValue(CharSequence name, byte type) { QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); beginColumnWrite(col, name); col.addNull(); + completeColumnWrite(); inProgressRowValueCount++; } @@ -761,7 +783,6 @@ private void rollbackCurrentRowToCommittedState() { currentTableBuffer.cancelCurrentRow(); currentTableBuffer.rollbackUncommittedColumns(); } - restorePendingFillColumnsFromInProgressColumns(); clearCachedTimestampColumns(); clearInProgressRow(); } @@ -772,8 +793,12 @@ private void commitCurrentRow() { } long estimate = 0; + long committedEstimateBeforeRow = 0; int targetRows = currentTableBuffer.getRowCount() + 1; if (trackDatagramEstimate) { + committedEstimateBeforeRow = currentTableBuffer.getRowCount() > 0 + ? committedDatagramEstimate + : estimateBaseForCurrentSchema(); estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows); if (estimate > maxDatagramSize) { if (currentTableBuffer.getRowCount() == 0) { @@ -781,6 +806,7 @@ private void commitCurrentRow() { } flushCommittedPrefixPreservingCurrentRow(); targetRows = currentTableBuffer.getRowCount() + 1; + committedEstimateBeforeRow = estimateBaseForCurrentSchema(); estimate = estimateCurrentDatagramSizeWithInProgressRow(targetRows); if (estimate > maxDatagramSize) { throw singleRowTooLarge(estimate); @@ -788,12 +814,15 @@ private void commitCurrentRow() { } } - currentTableBuffer.nextRow(rowFillColumns, rowFillColumnCount); - restorePendingFillColumnsFromInProgressColumns(); + currentTableBuffer.nextRow(); if (trackDatagramEstimate) { committedDatagramEstimate = estimate; } clearInProgressRow(); + if (trackDatagramEstimate) { + long rowDatagramGrowth = estimate - committedEstimateBeforeRow; + recordCommittedRowAndMaybeFlush(rowDatagramGrowth); + } } private void ensureNoInProgressRow(String operation) { @@ -831,6 +860,13 @@ private int encodeCommittedPrefixPayloadForUdp(QwpTableBuffer tableBuffer) { } private long estimateBaseForCurrentSchema() { + if (currentTableHeadroomState != null) { + long cachedEstimate = currentTableHeadroomState.getCachedBaseEstimate(currentTableBuffer.getColumnCount()); + if (cachedEstimate > -1) { + return cachedEstimate; + } + } + long estimate = HEADER_SIZE; int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName); estimate += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; @@ -853,6 +889,9 @@ private long estimateBaseForCurrentSchema() { estimate += 1; } } + if (currentTableHeadroomState != null) { + currentTableHeadroomState.cacheBaseEstimate(currentTableBuffer.getColumnCount(), estimate); + } return estimate; } @@ -860,14 +899,17 @@ private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) { long estimate = currentTableBuffer.getRowCount() > 0 ? committedDatagramEstimate : estimateBaseForCurrentSchema(); for (int i = 0; i < inProgressColumnCount; i++) { InProgressColumnState state = inProgressColumns[i]; - QwpTableBuffer.ColumnBuffer col = state.column; - estimate += estimateInProgressColumnPayload(state); - if (col.isNullable()) { + estimate += state.payloadEstimateDelta; + if (state.nullable) { estimate += bitmapBytes(targetRows) - bitmapBytes(state.sizeBefore); } } - for (int i = 0; i < rowFillColumnCount; i++) { - QwpTableBuffer.ColumnBuffer col = rowFillColumns[i]; + ensureStagedColumnMarkCapacity(currentTableBuffer.getColumnCount()); + for (int i = 0, columnCount = currentTableBuffer.getColumnCount(); i < columnCount; i++) { + if (stagedColumnMarks[i] == currentRowMark) { + continue; + } + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); int missing = targetRows - col.getSize(); if (col.isNullable()) { estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); @@ -878,7 +920,7 @@ private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) { return estimate; } - private long estimateInProgressColumnPayload(InProgressColumnState state) { + private static long estimateInProgressColumnPayload(InProgressColumnState state) { QwpTableBuffer.ColumnBuffer col = state.column; int valueCountBefore = state.valueCountBefore; int valueCountAfter = col.getValueCount(); @@ -899,36 +941,6 @@ private long estimateInProgressColumnPayload(InProgressColumnState state) { }; } - private void ensureRowFillColumnCapacity(int required) { - if (required <= rowFillColumns.length) { - return; - } - - int newCapacity = rowFillColumns.length; - while (newCapacity < required) { - newCapacity *= 2; - } - - QwpTableBuffer.ColumnBuffer[] newArr = new QwpTableBuffer.ColumnBuffer[newCapacity]; - System.arraycopy(rowFillColumns, 0, newArr, 0, rowFillColumnCount); - rowFillColumns = newArr; - } - - private void ensureRowFillPositionCapacity(int required) { - if (required <= rowFillColumnPositions.length) { - return; - } - - int newCapacity = rowFillColumnPositions.length; - while (newCapacity < required) { - newCapacity *= 2; - } - - int[] newArr = new int[newCapacity]; - System.arraycopy(rowFillColumnPositions, 0, newArr, 0, rowFillColumnPositions.length); - rowFillColumnPositions = newArr; - } - private void ensureInProgressColumnCapacity(int required) { if (required <= inProgressColumns.length) { return; @@ -959,14 +971,14 @@ private void ensureStagedColumnMarkCapacity(int required) { stagedColumnMarks = newArr; } - private long estimateArrayPayloadBytes(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { + private static long estimateArrayPayloadBytes(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { int shapeCount = col.getArrayShapeOffset() - state.arrayShapeOffsetBefore; int dataCount = col.getArrayDataOffset() - state.arrayDataOffsetBefore; int elementSize = col.getType() == TYPE_LONG_ARRAY ? Long.BYTES : Double.BYTES; return 1L + (long) shapeCount * Integer.BYTES + (long) dataCount * elementSize; } - private long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { + private static long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { int valueCountBefore = state.valueCountBefore; int valueCountAfter = col.getValueCount(); if (valueCountAfter == valueCountBefore) { @@ -1006,7 +1018,6 @@ private void flushCommittedRowsOfCurrentTable() { } sendWholeTableBuffer(currentTableName, currentTableBuffer); clearCachedTimestampColumns(); - rebuildPendingFillColumnsForCurrentTable(); resetCommittedDatagramEstimate(); } @@ -1024,7 +1035,6 @@ private void flushInternal() { sendWholeTableBuffer(tableName, tableBuffer); } clearTransientRowState(); - rebuildPendingFillColumnsForCurrentTable(); resetCommittedDatagramEstimate(); } @@ -1061,6 +1071,17 @@ private static int packedBytes(int valueCount) { return (valueCount + 7) / 8; } + private void recordCommittedRowAndMaybeFlush(long rowDatagramGrowth) { + if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0 || currentTableHeadroomState == null) { + return; + } + + currentTableHeadroomState.recordCommittedRow(currentTableBuffer.getColumnCount(), rowDatagramGrowth); + if (shouldFlushCommittedRowsAfterCommit()) { + flushCommittedRowsOfCurrentTable(); + } + } + private void resetCommittedDatagramEstimate() { committedDatagramEstimate = 0; } @@ -1073,7 +1094,6 @@ private void clearCachedTimestampColumns() { private void clearTransientRowState() { clearCachedTimestampColumns(); clearInProgressRow(); - clearPendingFillColumns(); } // Public test hooks because module boundaries prevent tests from sharing this package. @@ -1154,79 +1174,6 @@ private void flushCommittedPrefixPreservingCurrentRow() { } } - private void clearPendingFillColumns() { - for (int i = 0; i < rowFillColumnCount; i++) { - QwpTableBuffer.ColumnBuffer column = rowFillColumns[i]; - if (column != null) { - rowFillColumnPositions[column.getIndex()] = 0; - rowFillColumns[i] = null; - } - } - rowFillColumnCount = 0; - } - - private void rebuildPendingFillColumnsForCurrentTable() { - clearPendingFillColumns(); - if (currentTableBuffer == null) { - return; - } - - int columnCount = currentTableBuffer.getColumnCount(); - ensureRowFillColumnCapacity(columnCount); - ensureRowFillPositionCapacity(columnCount); - for (int i = 0; i < columnCount; i++) { - QwpTableBuffer.ColumnBuffer column = currentTableBuffer.getColumn(i); - rowFillColumns[rowFillColumnCount] = column; - rowFillColumnPositions[i] = rowFillColumnCount + 1; - rowFillColumnCount++; - } - } - - private void removePendingFillColumn(QwpTableBuffer.ColumnBuffer column) { - int columnIndex = column.getIndex(); - if (columnIndex >= rowFillColumnPositions.length) { - return; - } - - int position = rowFillColumnPositions[columnIndex] - 1; - if (position < 0) { - return; - } - - int lastIndex = rowFillColumnCount - 1; - QwpTableBuffer.ColumnBuffer lastColumn = rowFillColumns[lastIndex]; - rowFillColumns[lastIndex] = null; - rowFillColumnCount = lastIndex; - rowFillColumnPositions[columnIndex] = 0; - if (position != lastIndex) { - rowFillColumns[position] = lastColumn; - rowFillColumnPositions[lastColumn.getIndex()] = position + 1; - } - } - - private void restorePendingFillColumnsFromInProgressColumns() { - if (currentTableBuffer == null) { - return; - } - - int columnCount = currentTableBuffer.getColumnCount(); - ensureRowFillColumnCapacity(rowFillColumnCount + inProgressColumnCount); - ensureRowFillPositionCapacity(columnCount); - for (int i = 0; i < inProgressColumnCount; i++) { - QwpTableBuffer.ColumnBuffer column = inProgressColumns[i].column; - int columnIndex = column.getIndex(); - if (columnIndex >= columnCount || currentTableBuffer.getColumn(columnIndex) != column) { - continue; - } - if (rowFillColumnPositions[columnIndex] != 0) { - continue; - } - rowFillColumns[rowFillColumnCount] = column; - rowFillColumnPositions[columnIndex] = rowFillColumnCount + 1; - rowFillColumnCount++; - } - } - private void sendCommittedPrefix(CharSequence tableName, QwpTableBuffer tableBuffer) { int payloadLength = encodeCommittedPrefixPayloadForUdp(tableBuffer); sendEncodedPayload(tableName, payloadLength); @@ -1264,6 +1211,18 @@ private void sendWholeTableBuffer(CharSequence tableName, QwpTableBuffer tableBu tableBuffer.reset(); } + private boolean shouldFlushCommittedRowsAfterCommit() { + if (committedDatagramEstimate >= maxDatagramSize) { + return true; + } + + long predictedNextRowGrowth = currentTableHeadroomState.predictNextRowGrowth(); + if (predictedNextRowGrowth <= 0) { + return false; + } + return committedDatagramEstimate > maxDatagramSize - predictedNextRowGrowth; + } + private LineSenderException singleRowTooLarge(long estimate) { return new LineSenderException( "single row exceeds maximum datagram size (" + maxDatagramSize @@ -1320,6 +1279,8 @@ private static final class InProgressColumnState { private int arrayDataOffsetBefore; private int arrayShapeOffsetBefore; private QwpTableBuffer.ColumnBuffer column; + private boolean nullable; + private long payloadEstimateDelta; private int sizeBefore; private long stringDataSizeBefore; private int symbolDictionarySizeBefore; @@ -1327,10 +1288,14 @@ private static final class InProgressColumnState { void clear() { column = null; + nullable = false; + payloadEstimateDelta = 0; } void of(QwpTableBuffer.ColumnBuffer column) { this.column = column; + this.nullable = column.isNullable(); + this.payloadEstimateDelta = 0; this.sizeBefore = column.getSize(); this.valueCountBefore = column.getValueCount(); this.stringDataSizeBefore = column.getStringDataSize(); @@ -1339,6 +1304,10 @@ void of(QwpTableBuffer.ColumnBuffer column) { this.symbolDictionarySizeBefore = column.getSymbolDictionarySize(); } + void captureAfterWrite() { + this.payloadEstimateDelta = estimateInProgressColumnPayload(this); + } + void rebaseToEmptyTable() { this.sizeBefore = 0; this.valueCountBefore = 0; @@ -1346,6 +1315,51 @@ void rebaseToEmptyTable() { this.arrayShapeOffsetBefore = 0; this.arrayDataOffsetBefore = 0; this.symbolDictionarySizeBefore = 0; + captureAfterWrite(); + } + } + + private static final class TableHeadroomState { + private long cachedBaseEstimate = -1; + private int cachedBaseEstimateColumnCount = -1; + private int committedSampleCount; + private int schemaColumnCount = -1; + private long ewmaRowDatagramGrowth; + private long lastRowDatagramGrowth; + + void cacheBaseEstimate(int currentSchemaColumnCount, long estimate) { + cachedBaseEstimateColumnCount = currentSchemaColumnCount; + cachedBaseEstimate = estimate; + } + + long getCachedBaseEstimate(int currentSchemaColumnCount) { + return cachedBaseEstimateColumnCount == currentSchemaColumnCount ? cachedBaseEstimate : -1; + } + + long predictNextRowGrowth() { + if (committedSampleCount < 2) { + return 0; + } + return Math.max(lastRowDatagramGrowth, ewmaRowDatagramGrowth); + } + + void recordCommittedRow(int currentSchemaColumnCount, long rowDatagramGrowth) { + if (schemaColumnCount != currentSchemaColumnCount) { + committedSampleCount = 0; + ewmaRowDatagramGrowth = 0; + lastRowDatagramGrowth = 0; + schemaColumnCount = currentSchemaColumnCount; + } + + lastRowDatagramGrowth = rowDatagramGrowth; + if (committedSampleCount == 0) { + ewmaRowDatagramGrowth = rowDatagramGrowth; + } else { + ewmaRowDatagramGrowth += (rowDatagramGrowth - ewmaRowDatagramGrowth) >> ADAPTIVE_HEADROOM_EWMA_SHIFT; + } + if (committedSampleCount < Integer.MAX_VALUE) { + committedSampleCount++; + } } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 0abf8db..b663a10 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -147,6 +147,138 @@ public void testEstimateMatchesActualEncodedSize() throws Exception { }); } + @Test + public void testAdaptiveHeadroomFlushesCommittedRowsBeforeNextRowStarts() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String beta = repeat('b', 256); + String gamma = repeat('c', 256); + List rows = Arrays.asList( + row("t", sender -> sender.table("t") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(), + "x", 1L, + "s", alpha), + row("t", sender -> sender.table("t") + .longColumn("x", 2) + .stringColumn("s", beta) + .atNow(), + "x", 2L, + "s", beta), + row("t", sender -> sender.table("t") + .longColumn("x", 3) + .stringColumn("s", gamma) + .atNow(), + "x", 3L, + "s", gamma) + ); + + int oneRowPacket = fullPacketSize(rows.subList(0, 1)); + int twoRowPacket = fullPacketSize(rows.subList(0, 2)); + int maxDatagramSize = oneRowPacket + 16; + Assert.assertTrue("expected overflow boundary between rows 1 and 2", maxDatagramSize < twoRowPacket); + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("t") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(); + Assert.assertEquals(0, nf.sendCount); + + sender.longColumn("x", 2) + .stringColumn("s", beta) + .atNow(); + Assert.assertEquals("expected adaptive post-commit flush for row 2", 2, nf.sendCount); + + sender.longColumn("x", 3) + .stringColumn("s", gamma) + .atNow(); + Assert.assertEquals("expected row 3 to start on a fresh datagram", 3, nf.sendCount); + + sender.flush(); + } + + RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount); + Assert.assertEquals(3, result.sendCount); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(expectedRows(rows), decodeRows(nf.packets)); + }); + } + + @Test + public void testAdaptiveHeadroomStateIsPerTable() throws Exception { + assertMemoryLeak(() -> { + String alpha = repeat('a', 256); + String beta = repeat('b', 256); + List bigRows = Arrays.asList( + row("big", sender -> sender.table("big") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(), + "x", 1L, + "s", alpha), + row("big", sender -> sender.table("big") + .longColumn("x", 2) + .stringColumn("s", beta) + .atNow(), + "x", 2L, + "s", beta) + ); + List smallRows = Arrays.asList( + row("small", sender -> sender.table("small") + .longColumn("x", 10) + .atNow(), + "x", 10L), + row("small", sender -> sender.table("small") + .longColumn("x", 11) + .atNow(), + "x", 11L) + ); + + int oneBigRowPacket = fullPacketSize(bigRows.subList(0, 1)); + int twoBigRowPacket = fullPacketSize(bigRows); + int twoSmallRowPacket = fullPacketSize(smallRows); + int maxDatagramSize = oneBigRowPacket + 16; + Assert.assertTrue("expected overflow boundary between big rows", maxDatagramSize < twoBigRowPacket); + Assert.assertTrue("expected small rows to fit together", twoSmallRowPacket <= maxDatagramSize); + + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) { + sender.table("big") + .longColumn("x", 1) + .stringColumn("s", alpha) + .atNow(); + sender.longColumn("x", 2) + .stringColumn("s", beta) + .atNow(); + Assert.assertEquals(2, nf.sendCount); + + sender.table("small") + .longColumn("x", 10) + .atNow(); + Assert.assertEquals("big-table headroom must not flush small-table row 1", 2, nf.sendCount); + + sender.longColumn("x", 11) + .atNow(); + Assert.assertEquals("small-table rows should share a datagram", 2, nf.sendCount); + + sender.flush(); + } + + RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount); + Assert.assertEquals(3, result.sendCount); + assertPacketsWithinLimit(result, maxDatagramSize); + assertRowsEqual(Arrays.asList( + decodedRow("big", "x", 1L, "s", alpha), + decodedRow("big", "x", 2L, "s", beta), + decodedRow("small", "x", 10L), + decodedRow("small", "x", 11L) + ), decodeRows(nf.packets)); + }); + } + @Test public void testBoundedSenderNullableStringNullAcrossOverflowBoundaryPreservesRowsAndPacketLimit() throws Exception { assertMemoryLeak(() -> { From 45e662d96dff2b4644d6dc63faa6909e3cecc1e4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 10 Mar 2026 12:17:22 +0100 Subject: [PATCH 173/230] Auto-reorganize code --- .../main/java/io/questdb/client/Sender.java | 94 +- .../cutlass/http/client/WebSocketClient.java | 1 - .../cutlass/qwp/client/QwpUdpSender.java | 997 +++++++++--------- .../line/tcp/v4/QwpAllocationTestClient.java | 13 +- .../qwp/protocol/QwpTableBufferTest.java | 881 ++++++++-------- 5 files changed, 990 insertions(+), 996 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 297a35e..af1dbde 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -57,9 +57,9 @@ import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.security.PrivateKey; -import java.util.Base64; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Base64; import java.util.concurrent.TimeUnit; /** @@ -534,12 +534,12 @@ final class LineSenderBuilder { private static final int DEFAULT_AUTO_FLUSH_INTERVAL_MILLIS = 1_000; private static final int DEFAULT_AUTO_FLUSH_ROWS = 75_000; private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024; - private static final int DEFAULT_MAX_DATAGRAM_SIZE = 1400; private static final int DEFAULT_HTTP_PORT = 9000; private static final int DEFAULT_HTTP_TIMEOUT = 30_000; private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8; private static final int DEFAULT_MAXIMUM_BUFFER_CAPACITY = 100 * 1024 * 1024; private static final int DEFAULT_MAX_BACKOFF_MILLIS = 1_000; + private static final int DEFAULT_MAX_DATAGRAM_SIZE = 1400; private static final int DEFAULT_MAX_NAME_LEN = 127; private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method @@ -576,7 +576,6 @@ final class LineSenderBuilder { private int maxDatagramSize = PARAMETER_NOT_SET_EXPLICITLY; private int maxNameLength = PARAMETER_NOT_SET_EXPLICITLY; private int maximumBufferCapacity = PARAMETER_NOT_SET_EXPLICITLY; - private int multicastTtl = PARAMETER_NOT_SET_EXPLICITLY; private final HttpClientConfiguration httpClientConfiguration = new DefaultHttpClientConfiguration() { @Override public int getInitialRequestBufferSize() { @@ -599,6 +598,7 @@ public int getTimeout() { } }; private long minRequestThroughput = PARAMETER_NOT_SET_EXPLICITLY; + private int multicastTtl = PARAMETER_NOT_SET_EXPLICITLY; private String password; private PrivateKey privateKey; private int protocol = PARAMETER_NOT_SET_EXPLICITLY; @@ -942,19 +942,13 @@ public Sender build() { channel = tlsChannel; } try { - switch (protocolVersion) { - case PROTOCOL_VERSION_V1: - sender = new LineTcpSenderV1(channel, bufferCapacity, maxNameLength); - break; - case PROTOCOL_VERSION_V2: - sender = new LineTcpSenderV2(channel, bufferCapacity, maxNameLength); - break; - case PROTOCOL_VERSION_V3: - sender = new LineTcpSenderV3(channel, bufferCapacity, maxNameLength); - break; - default: - throw new LineSenderException("unknown protocol version [version=").put(protocolVersion).put("]"); - } + sender = switch (protocolVersion) { + case PROTOCOL_VERSION_V1 -> new LineTcpSenderV1(channel, bufferCapacity, maxNameLength); + case PROTOCOL_VERSION_V2 -> new LineTcpSenderV2(channel, bufferCapacity, maxNameLength); + case PROTOCOL_VERSION_V3 -> new LineTcpSenderV3(channel, bufferCapacity, maxNameLength); + default -> + throw new LineSenderException("unknown protocol version [version=").put(protocolVersion).put("]"); + }; } catch (Throwable t) { channel.close(); throw rethrow(t); @@ -1243,32 +1237,6 @@ public LineSenderBuilder maxBackoffMillis(int maxBackoffMillis) { return this; } - /** - * Set the maximum datagram size in bytes for UDP transport. Only valid for UDP transport. - *
      - * The practical limit depends on the network MTU (typically 1500 bytes for Ethernet). - *
      - * Default value: 1400 bytes - * - * @param maxDatagramSize maximum datagram size in bytes - * @return this instance for method chaining - */ - public LineSenderBuilder maxDatagramSize(int maxDatagramSize) { - if (this.maxDatagramSize != PARAMETER_NOT_SET_EXPLICITLY) { - throw new LineSenderException("max datagram size was already configured ") - .put("[maxDatagramSize=").put(this.maxDatagramSize).put("]"); - } - if (maxDatagramSize < 1) { - throw new LineSenderException("max datagram size must be positive ") - .put("[maxDatagramSize=").put(maxDatagramSize).put("]"); - } - if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_UDP) { - throw new LineSenderException("max datagram size is only supported for UDP transport"); - } - this.maxDatagramSize = maxDatagramSize; - return this; - } - /** * Set the maximum local buffer capacity in bytes. *
      @@ -1295,6 +1263,32 @@ public LineSenderBuilder maxBufferCapacity(int maximumBufferCapacity) { return this; } + /** + * Set the maximum datagram size in bytes for UDP transport. Only valid for UDP transport. + *
      + * The practical limit depends on the network MTU (typically 1500 bytes for Ethernet). + *
      + * Default value: 1400 bytes + * + * @param maxDatagramSize maximum datagram size in bytes + * @return this instance for method chaining + */ + public LineSenderBuilder maxDatagramSize(int maxDatagramSize) { + if (this.maxDatagramSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("max datagram size was already configured ") + .put("[maxDatagramSize=").put(this.maxDatagramSize).put("]"); + } + if (maxDatagramSize < 1) { + throw new LineSenderException("max datagram size must be positive ") + .put("[maxDatagramSize=").put(maxDatagramSize).put("]"); + } + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_UDP) { + throw new LineSenderException("max datagram size is only supported for UDP transport"); + } + this.maxDatagramSize = maxDatagramSize; + return this; + } + /** * Set the maximum length of a table or column name in bytes. * Matches the `cairo.max.file.name.length` setting in the server. @@ -1842,6 +1836,14 @@ private void tcp() { protocol = PROTOCOL_TCP; } + private void udp() { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol was already configured ") + .put("[protocol=").put(protocol).put("]"); + } + protocol = PROTOCOL_UDP; + } + private void validateParameters() { if (hosts.size() == 0) { throw new LineSenderException("questdb server address not set"); @@ -1985,14 +1987,6 @@ private void validateParameters() { } } - private void udp() { - if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { - throw new LineSenderException("protocol was already configured ") - .put("[protocol=").put(protocol).put("]"); - } - protocol = PROTOCOL_UDP; - } - private void websocket() { if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol was already configured ") diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index e57dbe0..443c7a2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -697,7 +697,6 @@ private int recvOrDie(long ptr, int len, int timeout) { } private int recvOrTimeout(long ptr, int len, int timeout) { - long startTime = System.nanoTime(); int n = socket.recv(ptr, len); if (n < 0) { throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 37b7638..b37e93f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -31,6 +31,7 @@ import io.questdb.client.cutlass.line.udp.UdpLineChannel; import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.network.NetworkFacade; import io.questdb.client.std.CharSequenceObjHashMap; import io.questdb.client.std.Chars; import io.questdb.client.std.Decimal128; @@ -39,7 +40,6 @@ import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.std.bytes.DirectByteSlice; -import io.questdb.client.network.NetworkFacade; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; @@ -65,44 +65,40 @@ */ public class QwpUdpSender implements Sender { private static final int ADAPTIVE_HEADROOM_EWMA_SHIFT = 2; - private static final int VARINT_INT_UPPER_BOUND = 5; private static final Logger LOG = LoggerFactory.getLogger(QwpUdpSender.class); - + private static final int VARINT_INT_UPPER_BOUND = 5; private final UdpLineChannel channel; private final QwpColumnWriter columnWriter = new QwpColumnWriter(); + private final NativeSegmentList datagramSegments = new NativeSegmentList(); private final NativeBufferWriter headerBuffer = new NativeBufferWriter(); private final int maxDatagramSize; private final SegmentedNativeBufferWriter payloadWriter = new SegmentedNativeBufferWriter(); - private final boolean trackDatagramEstimate; - private final NativeSegmentList datagramSegments = new NativeSegmentList(); - private final CharSequenceObjHashMap tableHeadroomStates; private final CharSequenceObjHashMap tableBuffers; - private InProgressColumnState[] inProgressColumns = new InProgressColumnState[8]; - + private final CharSequenceObjHashMap tableHeadroomStates; + private final boolean trackDatagramEstimate; private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; private boolean closed; private long committedDatagramEstimate; - private int inProgressRowValueCount; - private TableHeadroomState currentTableHeadroomState; + // monotonically increasing mark; incremented per row to invalidate stagedColumnMarks without clearing + private int currentRowMark = 1; private QwpTableBuffer currentTableBuffer; + private TableHeadroomState currentTableHeadroomState; private String currentTableName; - + private int inProgressColumnCount; + private InProgressColumnState[] inProgressColumns = new InProgressColumnState[8]; + private int inProgressRowValueCount; // prefix* arrays: per-column snapshots captured before the in-progress row, // used to encode and flush only the committed prefix when a row is still being built. // Indexed by column index. -1 means the column has no in-progress data. private int[] prefixArrayDataOffsetBefore = new int[8]; private int[] prefixArrayShapeOffsetBefore = new int[8]; + private int[] prefixSizeBefore = new int[8]; private long[] prefixStringDataSizeBefore = new long[8]; private int[] prefixSymbolDictionarySizeBefore = new int[8]; - private int[] prefixSizeBefore = new int[8]; private int[] prefixValueCountBefore = new int[8]; - // per-column marks to detect duplicate writes within a single row; compared against currentRowMark private int[] stagedColumnMarks = new int[8]; - // monotonically increasing mark; incremented per row to invalidate stagedColumnMarks without clearing - private int currentRowMark = 1; - private int inProgressColumnCount; public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl) { this(nf, interfaceIPv4, sendToAddress, port, ttl, 0); @@ -204,6 +200,16 @@ public void close() { } } + @TestOnly + public long committedDatagramEstimateForTest() { + return committedDatagramEstimate; + } + + @TestOnly + public QwpTableBuffer currentTableBufferForTest() { + return currentTableBuffer; + } + @Override public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) { @@ -432,6 +438,17 @@ public void reset() { tableHeadroomStates.clear(); } + // Public test hooks because module boundaries prevent tests from sharing this package. + @TestOnly + public void stageNullDoubleArrayForTest(CharSequence name) { + stageNullArrayColumnValue(name, TYPE_DOUBLE_ARRAY); + } + + @TestOnly + public void stageNullLongArrayForTest(CharSequence name) { + stageNullArrayColumnValue(name, TYPE_LONG_ARRAY); + } + @Override public Sender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); @@ -518,6 +535,118 @@ public Sender timestampColumn(CharSequence columnName, Instant value) { return this; } + private static int bitmapBytes(int size) { + return (size + 7) / 8; + } + + private static long estimateArrayPayloadBytes(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { + int shapeCount = col.getArrayShapeOffset() - state.arrayShapeOffsetBefore; + int dataCount = col.getArrayDataOffset() - state.arrayDataOffsetBefore; + int elementSize = col.getType() == TYPE_LONG_ARRAY ? Long.BYTES : Double.BYTES; + return 1L + (long) shapeCount * Integer.BYTES + (long) dataCount * elementSize; + } + + private static long estimateInProgressColumnPayload(InProgressColumnState state) { + QwpTableBuffer.ColumnBuffer col = state.column; + int valueCountBefore = state.valueCountBefore; + int valueCountAfter = col.getValueCount(); + if (valueCountAfter == valueCountBefore) { + return 0; + } + + return switch (col.getType()) { + case TYPE_BOOLEAN -> packedBytes(valueCountAfter) - packedBytes(valueCountBefore); + case TYPE_DECIMAL64 -> 8; + case TYPE_DECIMAL128 -> 16; + case TYPE_DECIMAL256 -> 32; + case TYPE_DOUBLE, TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> 8; + case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> estimateArrayPayloadBytes(col, state); + case TYPE_STRING, TYPE_VARCHAR -> 4L + (col.getStringDataSize() - state.stringDataSizeBefore); + case TYPE_SYMBOL -> estimateSymbolPayloadDelta(col, state); + default -> throw new LineSenderException("unsupported in-progress column type: " + col.getType()); + }; + } + + private static long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { + int valueCountBefore = state.valueCountBefore; + int valueCountAfter = col.getValueCount(); + if (valueCountAfter == valueCountBefore) { + return 0; + } + + int dictSizeBefore = state.symbolDictionarySizeBefore; + long dataAddress = col.getDataAddress(); + int idx = Unsafe.getUnsafe().getInt(dataAddress + (long) valueCountBefore * Integer.BYTES); + int dictSizeAfter = col.getSymbolDictionarySize(); + + if (dictSizeAfter == dictSizeBefore) { + return NativeBufferWriter.varintSize(idx); + } + + long delta = 0; + CharSequence value = col.getSymbolValue(idx); + int utf8Len = utf8Length(value); + delta += NativeBufferWriter.varintSize(utf8Len) + utf8Len; + delta += NativeBufferWriter.varintSize(dictSizeAfter) - NativeBufferWriter.varintSize(dictSizeBefore); + + if (dictSizeBefore > 0 && valueCountBefore > 0) { + int oldMax = dictSizeBefore - 1; + int newMax = dictSizeAfter - 1; + delta += (long) valueCountBefore * ( + NativeBufferWriter.varintSize(newMax) - NativeBufferWriter.varintSize(oldMax) + ); + } + + delta += NativeBufferWriter.varintSize(dictSizeAfter - 1); + return delta; + } + + private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) { + return switch (type) { + case TYPE_BOOLEAN -> packedBytes(valuesBefore + missing) - packedBytes(valuesBefore); + case TYPE_BYTE -> missing; + case TYPE_SHORT, TYPE_CHAR -> (long) missing * 2; + case TYPE_INT, TYPE_FLOAT -> (long) missing * 4; + case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> (long) missing * 8; + case TYPE_UUID -> (long) missing * 16; + case TYPE_LONG256 -> (long) missing * 32; + case TYPE_DECIMAL64 -> (long) missing * 8; + case TYPE_DECIMAL128 -> (long) missing * 16; + case TYPE_DECIMAL256 -> (long) missing * 32; + case TYPE_STRING, TYPE_VARCHAR -> (long) missing * 4; + case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> (long) missing * 5; + case TYPE_SYMBOL -> throw new IllegalStateException("symbol columns must be nullable"); + default -> 0; + }; + } + + private static int packedBytes(int valueCount) { + return (valueCount + 7) / 8; + } + + private static int utf8Length(CharSequence s) { + if (s == null) { + return 0; + } + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { + i++; + len += 4; + } else if (Character.isSurrogate(c)) { + len++; + } else { + len += 3; + } + } + return len; + } + private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean nullable) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn(name, type); if (col == null && currentTableBuffer.getRowCount() > 0) { @@ -538,17 +667,24 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, return col; } - private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence columnName) { - int columnIndex = column.getIndex(); - ensureStagedColumnMarkCapacity(columnIndex + 1); - if (stagedColumnMarks[columnIndex] == currentRowMark) { - if (columnName != null && columnName.isEmpty()) { - throw new LineSenderException("designated timestamp already set for current row"); - } - throw new LineSenderException("column '" + columnName + "' already set for current row"); + private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { + if (value instanceof double[] values) { + column.addDoubleArray(values); + return; } - stagedColumnMarks[columnIndex] = currentRowMark; - appendInProgressColumnState(column); + if (value instanceof double[][] values) { + column.addDoubleArray(values); + return; + } + if (value instanceof double[][][] values) { + column.addDoubleArray(values); + return; + } + if (value instanceof DoubleArray values) { + column.addDoubleArray(values); + return; + } + throw new LineSenderException("unsupported double array type"); } private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) { @@ -562,189 +698,81 @@ private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) { inProgressColumnCount++; } - private void completeColumnWrite() { - inProgressColumns[inProgressColumnCount - 1].captureAfterWrite(); - } - - private void stageBooleanColumnValue(CharSequence name, boolean value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); - beginColumnWrite(col, name); - col.addBoolean(value); - completeColumnWrite(); - inProgressRowValueCount++; + private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { + if (value instanceof long[] values) { + column.addLongArray(values); + return; + } + if (value instanceof long[][] values) { + column.addLongArray(values); + return; + } + if (value instanceof long[][][] values) { + column.addLongArray(values); + return; + } + if (value instanceof LongArray values) { + column.addLongArray(values); + return; + } + throw new LineSenderException("unsupported long array type"); } - private void stageDecimal128ColumnValue(CharSequence name, Decimal128 value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); - beginColumnWrite(col, name); - col.addDecimal128(value); - completeColumnWrite(); - inProgressRowValueCount++; + private void atMicros(long timestampMicros) { + if (inProgressRowValueCount == 0) { + throw new LineSenderException("no columns were provided"); + } + try { + stageDesignatedTimestampValue(timestampMicros, false); + commitCurrentRow(); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } } - private void stageDecimal256ColumnValue(CharSequence name, Decimal256 value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); - beginColumnWrite(col, name); - col.addDecimal256(value); - completeColumnWrite(); - inProgressRowValueCount++; + private void atNanos(long timestampNanos) { + if (inProgressRowValueCount == 0) { + throw new LineSenderException("no columns were provided"); + } + try { + stageDesignatedTimestampValue(timestampNanos, true); + commitCurrentRow(); + } catch (RuntimeException | Error e) { + rollbackCurrentRowToCommittedState(); + throw e; + } } - private void stageDecimal64ColumnValue(CharSequence name, Decimal64 value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); - beginColumnWrite(col, name); - col.addDecimal64(value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void stageDesignatedTimestampValue(long value, boolean nanos) { - QwpTableBuffer.ColumnBuffer col; - if (nanos) { - if (cachedTimestampNanosColumn == null) { - cachedTimestampNanosColumn = acquireColumn("", TYPE_TIMESTAMP_NANOS, true); - } - col = cachedTimestampNanosColumn; - } else { - if (cachedTimestampColumn == null) { - cachedTimestampColumn = acquireColumn("", TYPE_TIMESTAMP, true); + private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence columnName) { + int columnIndex = column.getIndex(); + ensureStagedColumnMarkCapacity(columnIndex + 1); + if (stagedColumnMarks[columnIndex] == currentRowMark) { + if (columnName != null && columnName.isEmpty()) { + throw new LineSenderException("designated timestamp already set for current row"); } - col = cachedTimestampColumn; - } - beginColumnWrite(col, ""); - col.addLong(value); - completeColumnWrite(); - } - - private void stageDoubleArrayColumnValue(CharSequence name, Object value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); - beginColumnWrite(col, name); - appendDoubleArrayValue(col, value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void stageDoubleColumnValue(CharSequence name, double value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); - beginColumnWrite(col, name); - col.addDouble(value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void stageLongArrayColumnValue(CharSequence name, Object value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); - beginColumnWrite(col, name); - appendLongArrayValue(col, value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void stageLongColumnValue(CharSequence name, long value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); - beginColumnWrite(col, name); - col.addLong(value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void stageStringColumnValue(CharSequence name, CharSequence value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); - beginColumnWrite(col, name); - col.addString(value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void stageSymbolColumnValue(CharSequence name, CharSequence value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); - beginColumnWrite(col, name); - col.addSymbol(value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void stageTimestampColumnValue(CharSequence name, byte type, long value) { - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); - beginColumnWrite(col, name); - col.addLong(value); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { - if (value instanceof double[] values) { - column.addDoubleArray(values); - return; - } - if (value instanceof double[][] values) { - column.addDoubleArray(values); - return; - } - if (value instanceof double[][][] values) { - column.addDoubleArray(values); - return; - } - if (value instanceof DoubleArray values) { - column.addDoubleArray(values); - return; - } - throw new LineSenderException("unsupported double array type"); - } - - private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { - if (value instanceof long[] values) { - column.addLongArray(values); - return; - } - if (value instanceof long[][] values) { - column.addLongArray(values); - return; - } - if (value instanceof long[][][] values) { - column.addLongArray(values); - return; - } - if (value instanceof LongArray values) { - column.addLongArray(values); - return; - } - throw new LineSenderException("unsupported long array type"); - } - - private void stageNullArrayColumnValue(CharSequence name, byte type) { - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); - beginColumnWrite(col, name); - col.addNull(); - completeColumnWrite(); - inProgressRowValueCount++; - } - - private void atMicros(long timestampMicros) { - if (inProgressRowValueCount == 0) { - throw new LineSenderException("no columns were provided"); - } - try { - stageDesignatedTimestampValue(timestampMicros, false); - commitCurrentRow(); - } catch (RuntimeException | Error e) { - rollbackCurrentRowToCommittedState(); - throw e; + throw new LineSenderException("column '" + columnName + "' already set for current row"); } + stagedColumnMarks[columnIndex] = currentRowMark; + appendInProgressColumnState(column); } - private void atNanos(long timestampNanos) { - if (inProgressRowValueCount == 0) { - throw new LineSenderException("no columns were provided"); + private void captureInProgressColumnPrefixState() { + int columnCount = currentTableBuffer.getColumnCount(); + ensurePrefixColumnCapacity(columnCount); + for (int i = 0; i < columnCount; i++) { + prefixSizeBefore[i] = -1; + prefixValueCountBefore[i] = -1; } - try { - stageDesignatedTimestampValue(timestampNanos, true); - commitCurrentRow(); - } catch (RuntimeException | Error e) { - rollbackCurrentRowToCommittedState(); - throw e; + for (int i = 0; i < inProgressColumnCount; i++) { + InProgressColumnState state = inProgressColumns[i]; + int columnIndex = state.column.getIndex(); + prefixSizeBefore[columnIndex] = state.sizeBefore; + prefixValueCountBefore[columnIndex] = state.valueCountBefore; + prefixStringDataSizeBefore[columnIndex] = state.stringDataSizeBefore; + prefixArrayShapeOffsetBefore[columnIndex] = state.arrayShapeOffsetBefore; + prefixArrayDataOffsetBefore[columnIndex] = state.arrayDataOffsetBefore; + prefixSymbolDictionarySizeBefore[columnIndex] = state.symbolDictionarySizeBefore; } } @@ -760,6 +788,11 @@ private void checkTableSelected() { } } + private void clearCachedTimestampColumns() { + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + private void clearInProgressRow() { inProgressRowValueCount = 0; for (int i = 0; i < inProgressColumnCount; i++) { @@ -778,11 +811,7 @@ private void clearInProgressRow() { inProgressColumnCount = 0; } - private void rollbackCurrentRowToCommittedState() { - if (currentTableBuffer != null) { - currentTableBuffer.cancelCurrentRow(); - currentTableBuffer.rollbackUncommittedColumns(); - } + private void clearTransientRowState() { clearCachedTimestampColumns(); clearInProgressRow(); } @@ -825,21 +854,8 @@ private void commitCurrentRow() { } } - private void ensureNoInProgressRow(String operation) { - if (hasInProgressRow()) { - throw new LineSenderException( - "Cannot " + operation + " while row is in progress. " - + "Use sender.at(), sender.atNow(), or sender.cancelRow() first." - ); - } - } - - private int encodeTablePayloadForUdp(QwpTableBuffer tableBuffer) { - payloadWriter.reset(); - columnWriter.setBuffer(payloadWriter); - columnWriter.encodeTable(tableBuffer, false, false, false); - payloadWriter.finish(); - return payloadWriter.getPosition(); + private void completeColumnWrite() { + inProgressColumns[inProgressColumnCount - 1].captureAfterWrite(); } private int encodeCommittedPrefixPayloadForUdp(QwpTableBuffer tableBuffer) { @@ -859,86 +875,12 @@ private int encodeCommittedPrefixPayloadForUdp(QwpTableBuffer tableBuffer) { return payloadWriter.getPosition(); } - private long estimateBaseForCurrentSchema() { - if (currentTableHeadroomState != null) { - long cachedEstimate = currentTableHeadroomState.getCachedBaseEstimate(currentTableBuffer.getColumnCount()); - if (cachedEstimate > -1) { - return cachedEstimate; - } - } - - long estimate = HEADER_SIZE; - int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName); - estimate += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; - estimate += VARINT_INT_UPPER_BOUND; - estimate += VARINT_INT_UPPER_BOUND; - estimate += 1; - - QwpColumnDef[] defs = currentTableBuffer.getColumnDefs(); - for (QwpColumnDef def : defs) { - int nameUtf8 = NativeBufferWriter.utf8Length(def.getName()); - estimate += NativeBufferWriter.varintSize(nameUtf8) + nameUtf8; - estimate += 1; - - byte type = def.getTypeCode(); - if (type == TYPE_STRING || type == TYPE_VARCHAR) { - estimate += 4; - } else if (type == TYPE_SYMBOL) { - estimate += 1; - } else if (type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128 || type == TYPE_DECIMAL256) { - estimate += 1; - } - } - if (currentTableHeadroomState != null) { - currentTableHeadroomState.cacheBaseEstimate(currentTableBuffer.getColumnCount(), estimate); - } - return estimate; - } - - private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) { - long estimate = currentTableBuffer.getRowCount() > 0 ? committedDatagramEstimate : estimateBaseForCurrentSchema(); - for (int i = 0; i < inProgressColumnCount; i++) { - InProgressColumnState state = inProgressColumns[i]; - estimate += state.payloadEstimateDelta; - if (state.nullable) { - estimate += bitmapBytes(targetRows) - bitmapBytes(state.sizeBefore); - } - } - ensureStagedColumnMarkCapacity(currentTableBuffer.getColumnCount()); - for (int i = 0, columnCount = currentTableBuffer.getColumnCount(); i < columnCount; i++) { - if (stagedColumnMarks[i] == currentRowMark) { - continue; - } - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); - int missing = targetRows - col.getSize(); - if (col.isNullable()) { - estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); - } else { - estimate += nonNullablePaddingCost(col.getType(), col.getValueCount(), missing); - } - } - return estimate; - } - - private static long estimateInProgressColumnPayload(InProgressColumnState state) { - QwpTableBuffer.ColumnBuffer col = state.column; - int valueCountBefore = state.valueCountBefore; - int valueCountAfter = col.getValueCount(); - if (valueCountAfter == valueCountBefore) { - return 0; - } - - return switch (col.getType()) { - case TYPE_BOOLEAN -> packedBytes(valueCountAfter) - packedBytes(valueCountBefore); - case TYPE_DECIMAL64 -> 8; - case TYPE_DECIMAL128 -> 16; - case TYPE_DECIMAL256 -> 32; - case TYPE_DOUBLE, TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> 8; - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> estimateArrayPayloadBytes(col, state); - case TYPE_STRING, TYPE_VARCHAR -> 4L + (col.getStringDataSize() - state.stringDataSizeBefore); - case TYPE_SYMBOL -> estimateSymbolPayloadDelta(col, state); - default -> throw new LineSenderException("unsupported in-progress column type: " + col.getType()); - }; + private int encodeTablePayloadForUdp(QwpTableBuffer tableBuffer) { + payloadWriter.reset(); + columnWriter.setBuffer(payloadWriter); + columnWriter.encodeTable(tableBuffer, false, false, false); + payloadWriter.finish(); + return payloadWriter.getPosition(); } private void ensureInProgressColumnCapacity(int required) { @@ -956,202 +898,107 @@ private void ensureInProgressColumnCapacity(int required) { inProgressColumns = newArr; } - private void ensureStagedColumnMarkCapacity(int required) { - if (required <= stagedColumnMarks.length) { - return; - } - - int newCapacity = stagedColumnMarks.length; - while (newCapacity < required) { - newCapacity *= 2; - } - - int[] newArr = new int[newCapacity]; - System.arraycopy(stagedColumnMarks, 0, newArr, 0, stagedColumnMarks.length); - stagedColumnMarks = newArr; - } - - private static long estimateArrayPayloadBytes(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { - int shapeCount = col.getArrayShapeOffset() - state.arrayShapeOffsetBefore; - int dataCount = col.getArrayDataOffset() - state.arrayDataOffsetBefore; - int elementSize = col.getType() == TYPE_LONG_ARRAY ? Long.BYTES : Double.BYTES; - return 1L + (long) shapeCount * Integer.BYTES + (long) dataCount * elementSize; - } - - private static long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { - int valueCountBefore = state.valueCountBefore; - int valueCountAfter = col.getValueCount(); - if (valueCountAfter == valueCountBefore) { - return 0; - } - - int dictSizeBefore = state.symbolDictionarySizeBefore; - long dataAddress = col.getDataAddress(); - int idx = Unsafe.getUnsafe().getInt(dataAddress + (long) valueCountBefore * Integer.BYTES); - int dictSizeAfter = col.getSymbolDictionarySize(); - - if (dictSizeAfter == dictSizeBefore) { - return NativeBufferWriter.varintSize(idx); - } - - long delta = 0; - CharSequence value = col.getSymbolValue(idx); - int utf8Len = utf8Length(value); - delta += NativeBufferWriter.varintSize(utf8Len) + utf8Len; - delta += NativeBufferWriter.varintSize(dictSizeAfter) - NativeBufferWriter.varintSize(dictSizeBefore); - - if (dictSizeBefore > 0 && valueCountBefore > 0) { - int oldMax = dictSizeBefore - 1; - int newMax = dictSizeAfter - 1; - delta += (long) valueCountBefore * ( - NativeBufferWriter.varintSize(newMax) - NativeBufferWriter.varintSize(oldMax) + private void ensureNoInProgressRow(String operation) { + if (hasInProgressRow()) { + throw new LineSenderException( + "Cannot " + operation + " while row is in progress. " + + "Use sender.at(), sender.atNow(), or sender.cancelRow() first." ); } - - delta += NativeBufferWriter.varintSize(dictSizeAfter - 1); - return delta; } - private void flushCommittedRowsOfCurrentTable() { - if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0) { + private void ensurePrefixColumnCapacity(int required) { + if (required <= prefixSizeBefore.length) { return; } - sendWholeTableBuffer(currentTableName, currentTableBuffer); - clearCachedTimestampColumns(); - resetCommittedDatagramEstimate(); - } - private void flushInternal() { - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } - sendWholeTableBuffer(tableName, tableBuffer); + int newCapacity = prefixSizeBefore.length; + while (newCapacity < required) { + newCapacity *= 2; } - clearTransientRowState(); - resetCommittedDatagramEstimate(); - } - - private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { - sendWholeTableBuffer(tableName, tableBuffer); - clearTransientRowState(); - resetCommittedDatagramEstimate(); - } - - private boolean hasInProgressRow() { - return inProgressColumnCount > 0; - } - private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) { - return switch (type) { - case TYPE_BOOLEAN -> packedBytes(valuesBefore + missing) - packedBytes(valuesBefore); - case TYPE_BYTE -> missing; - case TYPE_SHORT, TYPE_CHAR -> (long) missing * 2; - case TYPE_INT, TYPE_FLOAT -> (long) missing * 4; - case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> (long) missing * 8; - case TYPE_UUID -> (long) missing * 16; - case TYPE_LONG256 -> (long) missing * 32; - case TYPE_DECIMAL64 -> (long) missing * 8; - case TYPE_DECIMAL128 -> (long) missing * 16; - case TYPE_DECIMAL256 -> (long) missing * 32; - case TYPE_STRING, TYPE_VARCHAR -> (long) missing * 4; - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> (long) missing * 5; - case TYPE_SYMBOL -> throw new IllegalStateException("symbol columns must be nullable"); - default -> 0; - }; - } - - private static int packedBytes(int valueCount) { - return (valueCount + 7) / 8; + prefixSizeBefore = new int[newCapacity]; + prefixValueCountBefore = new int[newCapacity]; + prefixStringDataSizeBefore = new long[newCapacity]; + prefixArrayShapeOffsetBefore = new int[newCapacity]; + prefixArrayDataOffsetBefore = new int[newCapacity]; + prefixSymbolDictionarySizeBefore = new int[newCapacity]; } - private void recordCommittedRowAndMaybeFlush(long rowDatagramGrowth) { - if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0 || currentTableHeadroomState == null) { + private void ensureStagedColumnMarkCapacity(int required) { + if (required <= stagedColumnMarks.length) { return; } - currentTableHeadroomState.recordCommittedRow(currentTableBuffer.getColumnCount(), rowDatagramGrowth); - if (shouldFlushCommittedRowsAfterCommit()) { - flushCommittedRowsOfCurrentTable(); + int newCapacity = stagedColumnMarks.length; + while (newCapacity < required) { + newCapacity *= 2; } - } - - private void resetCommittedDatagramEstimate() { - committedDatagramEstimate = 0; - } - private void clearCachedTimestampColumns() { - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - } - - private void clearTransientRowState() { - clearCachedTimestampColumns(); - clearInProgressRow(); - } - - // Public test hooks because module boundaries prevent tests from sharing this package. - @TestOnly - public void stageNullDoubleArrayForTest(CharSequence name) { - stageNullArrayColumnValue(name, TYPE_DOUBLE_ARRAY); + int[] newArr = new int[newCapacity]; + System.arraycopy(stagedColumnMarks, 0, newArr, 0, stagedColumnMarks.length); + stagedColumnMarks = newArr; } - @TestOnly - public void stageNullLongArrayForTest(CharSequence name) { - stageNullArrayColumnValue(name, TYPE_LONG_ARRAY); - } + private long estimateBaseForCurrentSchema() { + if (currentTableHeadroomState != null) { + long cachedEstimate = currentTableHeadroomState.getCachedBaseEstimate(currentTableBuffer.getColumnCount()); + if (cachedEstimate > -1) { + return cachedEstimate; + } + } - @TestOnly - public QwpTableBuffer currentTableBufferForTest() { - return currentTableBuffer; - } + long estimate = HEADER_SIZE; + int tableNameUtf8 = NativeBufferWriter.utf8Length(currentTableName); + estimate += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; + estimate += VARINT_INT_UPPER_BOUND; + estimate += VARINT_INT_UPPER_BOUND; + estimate += 1; - @TestOnly - public long committedDatagramEstimateForTest() { - return committedDatagramEstimate; - } + QwpColumnDef[] defs = currentTableBuffer.getColumnDefs(); + for (QwpColumnDef def : defs) { + int nameUtf8 = NativeBufferWriter.utf8Length(def.getName()); + estimate += NativeBufferWriter.varintSize(nameUtf8) + nameUtf8; + estimate += 1; - private void captureInProgressColumnPrefixState() { - int columnCount = currentTableBuffer.getColumnCount(); - ensurePrefixColumnCapacity(columnCount); - for (int i = 0; i < columnCount; i++) { - prefixSizeBefore[i] = -1; - prefixValueCountBefore[i] = -1; - } - for (int i = 0; i < inProgressColumnCount; i++) { - InProgressColumnState state = inProgressColumns[i]; - int columnIndex = state.column.getIndex(); - prefixSizeBefore[columnIndex] = state.sizeBefore; - prefixValueCountBefore[columnIndex] = state.valueCountBefore; - prefixStringDataSizeBefore[columnIndex] = state.stringDataSizeBefore; - prefixArrayShapeOffsetBefore[columnIndex] = state.arrayShapeOffsetBefore; - prefixArrayDataOffsetBefore[columnIndex] = state.arrayDataOffsetBefore; - prefixSymbolDictionarySizeBefore[columnIndex] = state.symbolDictionarySizeBefore; + byte type = def.getTypeCode(); + if (type == TYPE_STRING || type == TYPE_VARCHAR) { + estimate += 4; + } else if (type == TYPE_SYMBOL) { + estimate += 1; + } else if (type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128 || type == TYPE_DECIMAL256) { + estimate += 1; + } + } + if (currentTableHeadroomState != null) { + currentTableHeadroomState.cacheBaseEstimate(currentTableBuffer.getColumnCount(), estimate); } + return estimate; } - private void ensurePrefixColumnCapacity(int required) { - if (required <= prefixSizeBefore.length) { - return; + private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) { + long estimate = currentTableBuffer.getRowCount() > 0 ? committedDatagramEstimate : estimateBaseForCurrentSchema(); + for (int i = 0; i < inProgressColumnCount; i++) { + InProgressColumnState state = inProgressColumns[i]; + estimate += state.payloadEstimateDelta; + if (state.nullable) { + estimate += bitmapBytes(targetRows) - bitmapBytes(state.sizeBefore); + } } - - int newCapacity = prefixSizeBefore.length; - while (newCapacity < required) { - newCapacity *= 2; + ensureStagedColumnMarkCapacity(currentTableBuffer.getColumnCount()); + for (int i = 0, columnCount = currentTableBuffer.getColumnCount(); i < columnCount; i++) { + if (stagedColumnMarks[i] == currentRowMark) { + continue; + } + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i); + int missing = targetRows - col.getSize(); + if (col.isNullable()) { + estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize()); + } else { + estimate += nonNullablePaddingCost(col.getType(), col.getValueCount(), missing); + } } - - prefixSizeBefore = new int[newCapacity]; - prefixValueCountBefore = new int[newCapacity]; - prefixStringDataSizeBefore = new long[newCapacity]; - prefixArrayShapeOffsetBefore = new int[newCapacity]; - prefixArrayDataOffsetBefore = new int[newCapacity]; - prefixSymbolDictionarySizeBefore = new int[newCapacity]; + return estimate; } private void flushCommittedPrefixPreservingCurrentRow() { @@ -1164,7 +1011,6 @@ private void flushCommittedPrefixPreservingCurrentRow() { currentTableBuffer.retainInProgressRow( prefixSizeBefore, prefixValueCountBefore, - prefixStringDataSizeBefore, prefixArrayShapeOffsetBefore, prefixArrayDataOffsetBefore ); @@ -1174,6 +1020,66 @@ private void flushCommittedPrefixPreservingCurrentRow() { } } + private void flushCommittedRowsOfCurrentTable() { + if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0) { + return; + } + sendWholeTableBuffer(currentTableName, currentTableBuffer); + clearCachedTimestampColumns(); + resetCommittedDatagramEstimate(); + } + + private void flushInternal() { + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; + } + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + sendWholeTableBuffer(tableName, tableBuffer); + } + clearTransientRowState(); + resetCommittedDatagramEstimate(); + } + + private void flushSingleTable(String tableName, QwpTableBuffer tableBuffer) { + sendWholeTableBuffer(tableName, tableBuffer); + clearTransientRowState(); + resetCommittedDatagramEstimate(); + } + + private boolean hasInProgressRow() { + return inProgressColumnCount > 0; + } + + private void recordCommittedRowAndMaybeFlush(long rowDatagramGrowth) { + if (currentTableBuffer == null || currentTableBuffer.getRowCount() == 0 || currentTableHeadroomState == null) { + return; + } + + currentTableHeadroomState.recordCommittedRow(currentTableBuffer.getColumnCount(), rowDatagramGrowth); + if (shouldFlushCommittedRowsAfterCommit()) { + flushCommittedRowsOfCurrentTable(); + } + } + + private void resetCommittedDatagramEstimate() { + committedDatagramEstimate = 0; + } + + private void rollbackCurrentRowToCommittedState() { + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + currentTableBuffer.rollbackUncommittedColumns(); + } + clearCachedTimestampColumns(); + clearInProgressRow(); + } + private void sendCommittedPrefix(CharSequence tableName, QwpTableBuffer tableBuffer) { int payloadLength = encodeCommittedPrefixPayloadForUdp(tableBuffer); sendEncodedPayload(tableName, payloadLength); @@ -1230,6 +1136,122 @@ private LineSenderException singleRowTooLarge(long estimate) { ); } + private void stageBooleanColumnValue(CharSequence name, boolean value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_BOOLEAN, false); + beginColumnWrite(col, name); + col.addBoolean(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDecimal128ColumnValue(CharSequence name, Decimal128 value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL128, true); + beginColumnWrite(col, name); + col.addDecimal128(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDecimal256ColumnValue(CharSequence name, Decimal256 value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL256, true); + beginColumnWrite(col, name); + col.addDecimal256(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDecimal64ColumnValue(CharSequence name, Decimal64 value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DECIMAL64, true); + beginColumnWrite(col, name); + col.addDecimal64(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDesignatedTimestampValue(long value, boolean nanos) { + QwpTableBuffer.ColumnBuffer col; + if (nanos) { + if (cachedTimestampNanosColumn == null) { + cachedTimestampNanosColumn = acquireColumn("", TYPE_TIMESTAMP_NANOS, true); + } + col = cachedTimestampNanosColumn; + } else { + if (cachedTimestampColumn == null) { + cachedTimestampColumn = acquireColumn("", TYPE_TIMESTAMP, true); + } + col = cachedTimestampColumn; + } + beginColumnWrite(col, ""); + col.addLong(value); + completeColumnWrite(); + } + + private void stageDoubleArrayColumnValue(CharSequence name, Object value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE_ARRAY, true); + beginColumnWrite(col, name); + appendDoubleArrayValue(col, value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageDoubleColumnValue(CharSequence name, double value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_DOUBLE, false); + beginColumnWrite(col, name); + col.addDouble(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageLongArrayColumnValue(CharSequence name, Object value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG_ARRAY, true); + beginColumnWrite(col, name); + appendLongArrayValue(col, value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageLongColumnValue(CharSequence name, long value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_LONG, false); + beginColumnWrite(col, name); + col.addLong(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageNullArrayColumnValue(CharSequence name, byte type) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); + beginColumnWrite(col, name); + col.addNull(); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageStringColumnValue(CharSequence name, CharSequence value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_STRING, true); + beginColumnWrite(col, name); + col.addString(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageSymbolColumnValue(CharSequence name, CharSequence value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, TYPE_SYMBOL, true); + beginColumnWrite(col, name); + col.addSymbol(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + + private void stageTimestampColumnValue(CharSequence name, byte type, long value) { + QwpTableBuffer.ColumnBuffer col = acquireColumn(name, type, true); + beginColumnWrite(col, name); + col.addLong(value); + completeColumnWrite(); + inProgressRowValueCount++; + } + private long toMicros(long value, ChronoUnit unit) { return switch (unit) { case NANOS -> value / 1000L; @@ -1243,33 +1265,6 @@ private long toMicros(long value, ChronoUnit unit) { }; } - private static int bitmapBytes(int size) { - return (size + 7) / 8; - } - - private static int utf8Length(CharSequence s) { - if (s == null) { - return 0; - } - int len = 0; - for (int i = 0, n = s.length(); i < n; i++) { - char c = s.charAt(i); - if (c < 0x80) { - len++; - } else if (c < 0x800) { - len += 2; - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { - i++; - len += 4; - } else if (Character.isSurrogate(c)) { - len++; - } else { - len += 3; - } - } - return len; - } - /** * Captures the state of a column buffer at the moment the in-progress row starts * writing to it. The snapshot allows the sender to compute incremental datagram @@ -1286,6 +1281,10 @@ private static final class InProgressColumnState { private int symbolDictionarySizeBefore; private int valueCountBefore; + void captureAfterWrite() { + this.payloadEstimateDelta = estimateInProgressColumnPayload(this); + } + void clear() { column = null; nullable = false; @@ -1304,10 +1303,6 @@ void of(QwpTableBuffer.ColumnBuffer column) { this.symbolDictionarySizeBefore = column.getSymbolDictionarySize(); } - void captureAfterWrite() { - this.payloadEstimateDelta = estimateInProgressColumnPayload(this); - } - void rebaseToEmptyTable() { this.sizeBefore = 0; this.valueCountBefore = 0; @@ -1323,9 +1318,9 @@ private static final class TableHeadroomState { private long cachedBaseEstimate = -1; private int cachedBaseEstimateColumnCount = -1; private int committedSampleCount; - private int schemaColumnCount = -1; private long ewmaRowDatagramGrowth; private long lastRowDatagramGrowth; + private int schemaColumnCount = -1; void cacheBaseEstimate(int currentSchemaColumnCount, long estimate) { cachedBaseEstimateColumnCount = currentSchemaColumnCount; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 7d0dc76..b3f5a98 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -182,9 +182,16 @@ public static void main(String[] args) { } } - private static Sender createSender(String protocol, String host, int port, - int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int maxDatagramSize) { + private static Sender createSender( + String protocol, + String host, + int port, + int batchSize, + int flushBytes, + long flushIntervalMs, + int inFlightWindow, + int maxDatagramSize + ) { switch (protocol) { case PROTOCOL_ILP_TCP: return Sender.builder(Sender.Transport.TCP) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index dc16bd2..02da14d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -163,6 +163,35 @@ public void testAddDoubleArrayNullOnNonNullableColumn() throws Exception { }); } + @Test + public void testAddDoubleArrayPayloadSupportsHigherDimensionalShape() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test"); + DoubleArray array = new DoubleArray(2, 1, 1, 2); + OffHeapAppendMemory payload = new OffHeapAppendMemory(128)) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + array.append(1.0).append(2.0).append(3.0).append(4.0); + array.appendToBufPtr(payload); + + col.addDoubleArrayPayload(payload.pageAddress(), payload.getAppendOffset()); + table.nextRow(); + + assertEquals(1, col.getValueCount()); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(4, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, shapes[1]); + assertEquals(1, shapes[2]); + assertEquals(2, shapes[3]); + + assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, readDoubleArraysLikeEncoder(col), 0.0); + } + }); + } + @Test public void testAddLongArrayNullOnNonNullableColumn() throws Exception { assertMemoryLeak(() -> { @@ -225,173 +254,76 @@ public void testAddSymbolNullOnNonNullableColumn() throws Exception { } @Test - public void testCancelRowTruncatesLateAddedColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - // Commit 3 rows with columns "a" (LONG, non-nullable) and "b" (STRING, nullable) - for (int i = 0; i < 3; i++) { - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(i); - table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v" + i); - table.nextRow(); - } - - // Start row 4: set "a" and "b", then create a NEW column "c" - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); - table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); - QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); - colC.addString("stale"); - - // Cancel the in-progress row - table.cancelCurrentRow(); - - // Column "c" was created during the in-progress row, so it must be fully cleared - assertEquals(0, colC.getSize()); - assertEquals(0, colC.getValueCount()); - - // Start row 4 again: set "a" and "b" only (not "c") - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); - table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); - table.nextRow(); - - // Column "c" should now have size == 4 (padded with nulls) and valueCount == 0 - assertEquals(4, colC.getSize()); - assertEquals(0, colC.getValueCount()); - - // All 4 rows of column "c" should be null - for (int i = 0; i < 4; i++) { - assertTrue("row " + i + " of column c should be null", colC.isNull(i)); - } - } - }); - } - - @Test - public void testCancelRowTruncatesLateAddedColumnWhenSizeEqualsRowCount() throws Exception { + public void testAddSymbolUtf8CancelRowRewindsDictionary() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("test")) { - // Commit exactly 1 row so rowCount == 1 table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); table.nextRow(); - // Start row 2: set "a", then create NEW column "c" with one value - // col_c.size will be 1, which equals rowCount — the edge case table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); - QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); - colC.addString("stale"); - - // Cancel the in-progress row + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true); + addSymbolUtf8(col, "stale"); table.cancelCurrentRow(); - // Column "c" had size == rowCount (1 == 1) but was still late-added - assertEquals(0, colC.getSize()); - assertEquals(0, colC.getValueCount()); - - // Start row 2 again without setting "c" table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + addSymbolUtf8(col, "fresh"); table.nextRow(); - // Column "c" should have 2 null rows - assertEquals(2, colC.getSize()); - assertEquals(0, colC.getValueCount()); - assertTrue(colC.isNull(0)); - assertTrue(colC.isNull(1)); + assertEquals(2, col.getSize()); + assertEquals(1, col.getValueCount()); + assertArrayEquals(new String[]{"fresh"}, col.getSymbolDictionary()); + assertEquals(0, Unsafe.getUnsafe().getInt(col.getDataAddress())); } }); } @Test - public void testNextRowWithPreparedMissingColumnsPadsListedColumns() throws Exception { + public void testAddSymbolUtf8RejectsInvalidUtf8() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); - QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); - QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_LONG, false); - - colA.addLong(10); - colB.addString("x"); - colC.addLong(100); - table.nextRow(); - - colA.addLong(20); - table.nextRow(new QwpTableBuffer.ColumnBuffer[]{colB, colC}, 2); - - assertEquals(2, colA.getSize()); - assertEquals(2, colA.getValueCount()); - assertEquals(2, colB.getSize()); - assertEquals(1, colB.getValueCount()); - assertFalse(colB.isNull(0)); - assertTrue(colB.isNull(1)); - assertEquals(2, colC.getSize()); - assertEquals(2, colC.getValueCount()); - assertEquals(100L, Unsafe.getUnsafe().getLong(colC.getDataAddress())); - assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(colC.getDataAddress() + Long.BYTES)); + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); + byte[] invalid = {(byte) 0xC3, 0x28}; + long ptr = copyToNative(invalid); + try { + try { + col.addSymbolUtf8(ptr, invalid.length); + fail("Expected CairoException"); + } catch (CairoException ex) { + assertTrue(ex.getFlyweightMessage().toString().contains("cannot convert invalid UTF-8 sequence")); + } + assertEquals(0, col.getSize()); + assertEquals(0, col.getValueCount()); + assertEquals(0, col.getSymbolDictionarySize()); + } finally { + Unsafe.free(ptr, invalid.length, MemoryTag.NATIVE_DEFAULT); + } } }); } @Test - public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exception { + public void testAddSymbolUtf8ReusesExistingDictionaryEntry() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer keep = table.getOrCreateColumn("keep", QwpConstants.TYPE_LONG, false); - QwpTableBuffer.ColumnBuffer drop = table.getOrCreateColumn("drop", QwpConstants.TYPE_STRING, true); - - for (int i = 0; i < 130; i++) { - keep.addLong(i); - if ((i & 1) == 0) { - drop.addString("v" + i); - } else { - drop.addNull(); - } - table.nextRow(); - } - - int keepSizeBefore = keep.getSize(); - int keepValueCountBefore = keep.getValueCount(); - long keepStringDataSizeBefore = keep.getStringDataSize(); - int keepArrayShapeOffsetBefore = keep.getArrayShapeOffset(); - int keepArrayDataOffsetBefore = keep.getArrayDataOffset(); - int keepIndex = keep.getIndex(); - - keep.addLong(130); - - int[] sizeBefore = {-1, -1}; - int[] valueCountBefore = {-1, -1}; - long[] stringDataSizeBefore = new long[2]; - int[] arrayShapeOffsetBefore = new int[2]; - int[] arrayDataOffsetBefore = new int[2]; - - sizeBefore[keepIndex] = keepSizeBefore; - valueCountBefore[keepIndex] = keepValueCountBefore; - stringDataSizeBefore[keepIndex] = keepStringDataSizeBefore; - arrayShapeOffsetBefore[keepIndex] = keepArrayShapeOffsetBefore; - arrayDataOffsetBefore[keepIndex] = keepArrayDataOffsetBefore; + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); + addSymbolUtf8(col, "東京"); + table.nextRow(); - table.retainInProgressRow( - sizeBefore, - valueCountBefore, - stringDataSizeBefore, - arrayShapeOffsetBefore, - arrayDataOffsetBefore - ); + addSymbolUtf8(col, "東京"); + table.nextRow(); - assertEquals(0, table.getRowCount()); + addSymbolUtf8(col, "Αθηνα"); + table.nextRow(); - assertEquals(1, keep.getSize()); - assertEquals(1, keep.getValueCount()); - assertEquals(130L, Unsafe.getUnsafe().getLong(keep.getDataAddress())); + assertEquals(3, col.getSize()); + assertEquals(3, col.getValueCount()); + assertEquals(2, col.getSymbolDictionarySize()); + assertArrayEquals(new String[]{"東京", "Αθηνα"}, col.getSymbolDictionary()); - assertEquals(0, drop.getSize()); - assertEquals(0, drop.getValueCount()); - assertEquals(0, drop.getStringDataSize()); - assertFalse(drop.isNull(0)); - assertFalse(drop.isNull(63)); - assertFalse(drop.isNull(64)); - assertFalse(drop.isNull(129)); - assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress())); - assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + Long.BYTES)); - assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + 2L * Long.BYTES)); - assertEquals(0, Unsafe.getUnsafe().getInt(drop.getStringOffsetsAddress())); + long dataAddress = col.getDataAddress(); + assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress)); + assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress + 4)); + assertEquals(1, Unsafe.getUnsafe().getInt(dataAddress + 8)); } }); } @@ -475,130 +407,55 @@ public void testCancelRowResetsSymbolDictOnLateAddedColumn() throws Exception { } @Test - public void testAddSymbolUtf8ReusesExistingDictionaryEntry() throws Exception { + public void testCancelRowRewindsDoubleArrayOffsets() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); - addSymbolUtf8(col, "東京"); + // Row 0: committed with [1.0, 2.0] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{1.0, 2.0}); table.nextRow(); - addSymbolUtf8(col, "東京"); + // Row 1: committed with [3.0, 4.0] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{3.0, 4.0}); table.nextRow(); - addSymbolUtf8(col, "Αθηνα"); + // Start row 2 with [5.0, 6.0] — then cancel it + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{5.0, 6.0}); + table.cancelCurrentRow(); + + // Add replacement row 2 with [7.0, 8.0] + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + col.addDoubleArray(new double[]{7.0, 8.0}); table.nextRow(); - assertEquals(3, col.getSize()); + assertEquals(3, table.getRowCount()); assertEquals(3, col.getValueCount()); - assertEquals(2, col.getSymbolDictionarySize()); - assertArrayEquals(new String[]{"東京", "Αθηνα"}, col.getSymbolDictionary()); - long dataAddress = col.getDataAddress(); - assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress)); - assertEquals(0, Unsafe.getUnsafe().getInt(dataAddress + 4)); - assertEquals(1, Unsafe.getUnsafe().getInt(dataAddress + 8)); + // Walk the arrays exactly as the encoder would + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals( + new double[]{1.0, 2.0, 3.0, 4.0, 7.0, 8.0}, + encoded, + 0.0 + ); } }); } @Test - public void testAddSymbolUtf8CancelRowRewindsDictionary() throws Exception { + public void testCancelRowRewindsLongArrayOffsets() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("test")) { - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + // Row 0: committed with [10, 20] + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{10, 20}); table.nextRow(); - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true); - addSymbolUtf8(col, "stale"); - table.cancelCurrentRow(); - - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); - addSymbolUtf8(col, "fresh"); - table.nextRow(); - - assertEquals(2, col.getSize()); - assertEquals(1, col.getValueCount()); - assertArrayEquals(new String[]{"fresh"}, col.getSymbolDictionary()); - assertEquals(0, Unsafe.getUnsafe().getInt(col.getDataAddress())); - } - }); - } - - @Test - public void testAddSymbolUtf8RejectsInvalidUtf8() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); - byte[] invalid = {(byte) 0xC3, 0x28}; - long ptr = copyToNative(invalid); - try { - try { - col.addSymbolUtf8(ptr, invalid.length); - fail("Expected CairoException"); - } catch (CairoException ex) { - assertTrue(ex.getFlyweightMessage().toString().contains("cannot convert invalid UTF-8 sequence")); - } - assertEquals(0, col.getSize()); - assertEquals(0, col.getValueCount()); - assertEquals(0, col.getSymbolDictionarySize()); - } finally { - Unsafe.free(ptr, invalid.length, MemoryTag.NATIVE_DEFAULT); - } - } - }); - } - - @Test - public void testCancelRowRewindsDoubleArrayOffsets() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - // Row 0: committed with [1.0, 2.0] - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{1.0, 2.0}); - table.nextRow(); - - // Row 1: committed with [3.0, 4.0] - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{3.0, 4.0}); - table.nextRow(); - - // Start row 2 with [5.0, 6.0] — then cancel it - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{5.0, 6.0}); - table.cancelCurrentRow(); - - // Add replacement row 2 with [7.0, 8.0] - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); - col.addDoubleArray(new double[]{7.0, 8.0}); - table.nextRow(); - - assertEquals(3, table.getRowCount()); - assertEquals(3, col.getValueCount()); - - // Walk the arrays exactly as the encoder would - double[] encoded = readDoubleArraysLikeEncoder(col); - assertArrayEquals( - new double[]{1.0, 2.0, 3.0, 4.0, 7.0, 8.0}, - encoded, - 0.0 - ); - } - }); - } - - @Test - public void testCancelRowRewindsLongArrayOffsets() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - // Row 0: committed with [10, 20] - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - col.addLongArray(new long[]{10, 20}); - table.nextRow(); - - // Start row 1 with [30, 40] — then cancel it - col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - col.addLongArray(new long[]{30, 40}); + // Start row 1 with [30, 40] — then cancel it + col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + col.addLongArray(new long[]{30, 40}); table.cancelCurrentRow(); // Add replacement row 1 with [50, 60] @@ -660,30 +517,76 @@ public void testCancelRowRewindsMultiDimArrayOffsets() throws Exception { } @Test - public void testAddDoubleArrayPayloadSupportsHigherDimensionalShape() throws Exception { + public void testCancelRowTruncatesLateAddedColumn() throws Exception { assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test"); - DoubleArray array = new DoubleArray(2, 1, 1, 2); - OffHeapAppendMemory payload = new OffHeapAppendMemory(128)) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Commit 3 rows with columns "a" (LONG, non-nullable) and "b" (STRING, nullable) + for (int i = 0; i < 3; i++) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(i); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v" + i); + table.nextRow(); + } - array.append(1.0).append(2.0).append(3.0).append(4.0); - array.appendToBufPtr(payload); + // Start row 4: set "a" and "b", then create a NEW column "c" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); + colC.addString("stale"); - col.addDoubleArrayPayload(payload.pageAddress(), payload.getAppendOffset()); + // Cancel the in-progress row + table.cancelCurrentRow(); + + // Column "c" was created during the in-progress row, so it must be fully cleared + assertEquals(0, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // Start row 4 again: set "a" and "b" only (not "c") + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(3); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true).addString("v3"); table.nextRow(); - assertEquals(1, col.getValueCount()); + // Column "c" should now have size == 4 (padded with nulls) and valueCount == 0 + assertEquals(4, colC.getSize()); + assertEquals(0, colC.getValueCount()); - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - assertEquals(4, dims[0]); - assertEquals(2, shapes[0]); - assertEquals(1, shapes[1]); - assertEquals(1, shapes[2]); - assertEquals(2, shapes[3]); + // All 4 rows of column "c" should be null + for (int i = 0; i < 4; i++) { + assertTrue("row " + i + " of column c should be null", colC.isNull(i)); + } + } + }); + } - assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, readDoubleArraysLikeEncoder(col), 0.0); + @Test + public void testCancelRowTruncatesLateAddedColumnWhenSizeEqualsRowCount() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + // Commit exactly 1 row so rowCount == 1 + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + // Start row 2: set "a", then create NEW column "c" with one value + // col_c.size will be 1, which equals rowCount — the edge case + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_STRING, true); + colC.addString("stale"); + + // Cancel the in-progress row + table.cancelCurrentRow(); + + // Column "c" had size == rowCount (1 == 1) but was still late-added + assertEquals(0, colC.getSize()); + assertEquals(0, colC.getValueCount()); + + // Start row 2 again without setting "c" + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + table.nextRow(); + + // Column "c" should have 2 null rows + assertEquals(2, colC.getSize()); + assertEquals(0, colC.getValueCount()); + assertTrue(colC.isNull(0)); + assertTrue(colC.isNull(1)); } }); } @@ -726,39 +629,6 @@ public void testDoubleArrayWrapperMultipleRows() throws Exception { }); } - @Test - public void testLongArrayWrapperMultipleRows() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test"); - LongArray arr = new LongArray(3)) { - QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - - arr.append(10).append(20).append(30); - col.addLongArray(arr); - table.nextRow(); - - arr.append(40).append(50).append(60); - col.addLongArray(arr); - table.nextRow(); - - arr.append(70).append(80).append(90); - col.addLongArray(arr); - table.nextRow(); - - assertEquals(3, col.getValueCount()); - long[] encoded = readLongArraysLikeEncoder(col); - assertArrayEquals(new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90}, encoded); - - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - for (int i = 0; i < 3; i++) { - assertEquals(1, dims[i]); - assertEquals(3, shapes[i]); - } - } - }); - } - @Test public void testDoubleArrayWrapperShrinkingSize() throws Exception { assertMemoryLeak(() -> { @@ -837,6 +707,183 @@ public void testDoubleArrayWrapperVaryingDimensionality() throws Exception { }); } + @Test + public void testGetExistingColumnReturnsNullWithoutCreatingColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + colA.addLong(1); + table.nextRow(); + + assertNull(table.getExistingColumn("missing", QwpConstants.TYPE_STRING)); + assertEquals(1, table.getColumnCount()); + + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + assertNotNull(colB); + assertEquals(2, table.getColumnCount()); + } + }); + } + + @Test + public void testGetExistingColumnReturnsOrderedColumnsAcrossRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + + assertSame(colA, existingA); + assertSame(colB, existingB); + + existingA.addLong(2); + existingB.addString("y"); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(2, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnReturnsOutOfOrderColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + + assertSame(colB, existingB); + assertSame(colA, existingA); + + existingB.addString("y"); + existingA.addLong(2); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, colA.getSize()); + assertEquals(2, colA.getValueCount()); + assertEquals(2, colB.getSize()); + assertEquals(2, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnTypeMismatchOnHashPathThrows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + try { + table.getExistingColumn("b", QwpConstants.TYPE_LONG); + fail("Expected LineSenderException for hash-path type mismatch"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Column type mismatch")); + assertTrue(e.getMessage().contains("column 'b'")); + } + } + }); + } + + @Test + public void testGetExistingColumnTypeMismatchOnOrderedPathThrows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + table.nextRow(); + + try { + table.getExistingColumn("a", QwpConstants.TYPE_STRING); + fail("Expected LineSenderException for ordered-path type mismatch"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Column type mismatch")); + assertTrue(e.getMessage().contains("column 'a'")); + } + } + }); + } + + @Test + public void testGetExistingColumnWorksAfterReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); + colA.addLong(1); + colB.addString("x"); + table.nextRow(); + + table.reset(); + + QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); + QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + + assertSame(colA, existingA); + assertSame(colB, existingB); + + existingA.addLong(2); + existingB.addString("y"); + table.nextRow(); + + assertEquals(1, table.getRowCount()); + assertEquals(1, colA.getSize()); + assertEquals(1, colA.getValueCount()); + assertEquals(1, colB.getSize()); + assertEquals(1, colB.getValueCount()); + } + }); + } + + @Test + public void testGetExistingColumnWorksForLateAddedColumnAfterCancelRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + table.nextRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(2); + QwpTableBuffer.ColumnBuffer late = table.getOrCreateColumn("late", QwpConstants.TYPE_STRING, true); + late.addString("stale"); + table.cancelCurrentRow(); + + QwpTableBuffer.ColumnBuffer existingLate = table.getExistingColumn("late", QwpConstants.TYPE_STRING); + assertSame(late, existingLate); + assertEquals(0, existingLate.getSize()); + assertEquals(0, existingLate.getValueCount()); + + table.getExistingColumn("a", QwpConstants.TYPE_LONG).addLong(2); + table.nextRow(); + + assertEquals(2, table.getRowCount()); + assertEquals(2, existingLate.getSize()); + assertEquals(0, existingLate.getValueCount()); + assertTrue(existingLate.isNull(0)); + assertTrue(existingLate.isNull(1)); + } + }); + } + @Test public void testGetOrCreateColumnConflictingTypeFastPath() throws Exception { assertMemoryLeak(() -> { @@ -943,180 +990,150 @@ public void testLongArrayShrinkingSize() throws Exception { } @Test - public void testGetExistingColumnReturnsOrderedColumnsAcrossRows() throws Exception { + public void testLongArrayWrapperMultipleRows() throws Exception { assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); - QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); - colA.addLong(1); - colB.addString("x"); - table.nextRow(); + try (QwpTableBuffer table = new QwpTableBuffer("test"); + LongArray arr = new LongArray(3)) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); - QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); - QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + arr.append(10).append(20).append(30); + col.addLongArray(arr); + table.nextRow(); - assertSame(colA, existingA); - assertSame(colB, existingB); + arr.append(40).append(50).append(60); + col.addLongArray(arr); + table.nextRow(); - existingA.addLong(2); - existingB.addString("y"); + arr.append(70).append(80).append(90); + col.addLongArray(arr); table.nextRow(); - assertEquals(2, table.getRowCount()); - assertEquals(2, colA.getSize()); - assertEquals(2, colA.getValueCount()); - assertEquals(2, colB.getSize()); - assertEquals(2, colB.getValueCount()); + assertEquals(3, col.getValueCount()); + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90}, encoded); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + for (int i = 0; i < 3; i++) { + assertEquals(1, dims[i]); + assertEquals(3, shapes[i]); + } } }); } @Test - public void testGetExistingColumnReturnsOutOfOrderColumns() throws Exception { + public void testNextRowWithPreparedMissingColumnsPadsListedColumns() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("test")) { QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); - colA.addLong(1); + QwpTableBuffer.ColumnBuffer colC = table.getOrCreateColumn("c", QwpConstants.TYPE_LONG, false); + + colA.addLong(10); colB.addString("x"); + colC.addLong(100); table.nextRow(); - QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); - QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); - - assertSame(colB, existingB); - assertSame(colA, existingA); - - existingB.addString("y"); - existingA.addLong(2); - table.nextRow(); + colA.addLong(20); + table.nextRow(new QwpTableBuffer.ColumnBuffer[]{colB, colC}, 2); - assertEquals(2, table.getRowCount()); assertEquals(2, colA.getSize()); assertEquals(2, colA.getValueCount()); assertEquals(2, colB.getSize()); - assertEquals(2, colB.getValueCount()); - } - }); - } - - @Test - public void testGetExistingColumnReturnsNullWithoutCreatingColumn() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); - colA.addLong(1); - table.nextRow(); - - assertNull(table.getExistingColumn("missing", QwpConstants.TYPE_STRING)); - assertEquals(1, table.getColumnCount()); - - QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); - assertNotNull(colB); - assertEquals(2, table.getColumnCount()); + assertEquals(1, colB.getValueCount()); + assertFalse(colB.isNull(0)); + assertTrue(colB.isNull(1)); + assertEquals(2, colC.getSize()); + assertEquals(2, colC.getValueCount()); + assertEquals(100L, Unsafe.getUnsafe().getLong(colC.getDataAddress())); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(colC.getDataAddress() + Long.BYTES)); } }); } @Test - public void testGetExistingColumnTypeMismatchOnOrderedPathThrows() throws Exception { + public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); - table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); - colA.addLong(1); - table.nextRow(); + QwpTableBuffer.ColumnBuffer keep = table.getOrCreateColumn("keep", QwpConstants.TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer drop = table.getOrCreateColumn("drop", QwpConstants.TYPE_STRING, true); - try { - table.getExistingColumn("a", QwpConstants.TYPE_STRING); - fail("Expected LineSenderException for ordered-path type mismatch"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Column type mismatch")); - assertTrue(e.getMessage().contains("column 'a'")); + for (int i = 0; i < 130; i++) { + keep.addLong(i); + if ((i & 1) == 0) { + drop.addString("v" + i); + } else { + drop.addNull(); + } + table.nextRow(); } - } - }); - } - @Test - public void testGetExistingColumnTypeMismatchOnHashPathThrows() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); - QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); - colA.addLong(1); - colB.addString("x"); - table.nextRow(); + int keepSizeBefore = keep.getSize(); + int keepValueCountBefore = keep.getValueCount(); + long keepStringDataSizeBefore = keep.getStringDataSize(); + int keepArrayShapeOffsetBefore = keep.getArrayShapeOffset(); + int keepArrayDataOffsetBefore = keep.getArrayDataOffset(); + int keepIndex = keep.getIndex(); - try { - table.getExistingColumn("b", QwpConstants.TYPE_LONG); - fail("Expected LineSenderException for hash-path type mismatch"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Column type mismatch")); - assertTrue(e.getMessage().contains("column 'b'")); - } - } - }); - } + keep.addLong(130); - @Test - public void testGetExistingColumnWorksAfterReset() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer colA = table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false); - QwpTableBuffer.ColumnBuffer colB = table.getOrCreateColumn("b", QwpConstants.TYPE_STRING, true); - colA.addLong(1); - colB.addString("x"); - table.nextRow(); + int[] sizeBefore = {-1, -1}; + int[] valueCountBefore = {-1, -1}; + long[] stringDataSizeBefore = new long[2]; + int[] arrayShapeOffsetBefore = new int[2]; + int[] arrayDataOffsetBefore = new int[2]; - table.reset(); + sizeBefore[keepIndex] = keepSizeBefore; + valueCountBefore[keepIndex] = keepValueCountBefore; + stringDataSizeBefore[keepIndex] = keepStringDataSizeBefore; + arrayShapeOffsetBefore[keepIndex] = keepArrayShapeOffsetBefore; + arrayDataOffsetBefore[keepIndex] = keepArrayDataOffsetBefore; - QwpTableBuffer.ColumnBuffer existingA = table.getExistingColumn("a", QwpConstants.TYPE_LONG); - QwpTableBuffer.ColumnBuffer existingB = table.getExistingColumn("b", QwpConstants.TYPE_STRING); + table.retainInProgressRow( + sizeBefore, + valueCountBefore, + arrayShapeOffsetBefore, + arrayDataOffsetBefore + ); - assertSame(colA, existingA); - assertSame(colB, existingB); + assertEquals(0, table.getRowCount()); - existingA.addLong(2); - existingB.addString("y"); - table.nextRow(); + assertEquals(1, keep.getSize()); + assertEquals(1, keep.getValueCount()); + assertEquals(130L, Unsafe.getUnsafe().getLong(keep.getDataAddress())); - assertEquals(1, table.getRowCount()); - assertEquals(1, colA.getSize()); - assertEquals(1, colA.getValueCount()); - assertEquals(1, colB.getSize()); - assertEquals(1, colB.getValueCount()); + assertEquals(0, drop.getSize()); + assertEquals(0, drop.getValueCount()); + assertEquals(0, drop.getStringDataSize()); + assertFalse(drop.isNull(0)); + assertFalse(drop.isNull(63)); + assertFalse(drop.isNull(64)); + assertFalse(drop.isNull(129)); + assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress())); + assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + Long.BYTES)); + assertEquals(0, Unsafe.getUnsafe().getLong(drop.getNullBitmapAddress() + 2L * Long.BYTES)); + assertEquals(0, Unsafe.getUnsafe().getInt(drop.getStringOffsetsAddress())); } }); } - @Test - public void testGetExistingColumnWorksForLateAddedColumnAfterCancelRow() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("test")) { - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); - table.nextRow(); - - table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(2); - QwpTableBuffer.ColumnBuffer late = table.getOrCreateColumn("late", QwpConstants.TYPE_STRING, true); - late.addString("stale"); - table.cancelCurrentRow(); - - QwpTableBuffer.ColumnBuffer existingLate = table.getExistingColumn("late", QwpConstants.TYPE_STRING); - assertSame(late, existingLate); - assertEquals(0, existingLate.getSize()); - assertEquals(0, existingLate.getValueCount()); - - table.getExistingColumn("a", QwpConstants.TYPE_LONG).addLong(2); - table.nextRow(); + private static void addSymbolUtf8(QwpTableBuffer.ColumnBuffer col, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + long ptr = copyToNative(bytes); + try { + col.addSymbolUtf8(ptr, bytes.length); + } finally { + Unsafe.free(ptr, bytes.length, MemoryTag.NATIVE_DEFAULT); + } + } - assertEquals(2, table.getRowCount()); - assertEquals(2, existingLate.getSize()); - assertEquals(0, existingLate.getValueCount()); - assertTrue(existingLate.isNull(0)); - assertTrue(existingLate.isNull(1)); - } - }); + private static long copyToNative(byte[] bytes) { + long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT); + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, bytes[i]); + } + return ptr; } /** @@ -1196,22 +1213,4 @@ private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) } return result; } - - private static void addSymbolUtf8(QwpTableBuffer.ColumnBuffer col, String value) { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - long ptr = copyToNative(bytes); - try { - col.addSymbolUtf8(ptr, bytes.length); - } finally { - Unsafe.free(ptr, bytes.length, MemoryTag.NATIVE_DEFAULT); - } - } - - private static long copyToNative(byte[] bytes) { - long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT); - for (int i = 0; i < bytes.length; i++) { - Unsafe.getUnsafe().putByte(ptr + i, bytes[i]); - } - return ptr; - } } From 7306114a9afe1aea977af4d4b534c0b3cf9d1778 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 10 Mar 2026 12:17:39 +0100 Subject: [PATCH 174/230] Remove LONG array support in QwpTableBuffer + auto-reorganize --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 541 +++++++++--------- 1 file changed, 262 insertions(+), 279 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index e4c46f4..185d4a5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -152,16 +152,6 @@ public int getColumnCount() { return columns.size(); } - /** - * Returns an existing column with the given name and type, or {@code null} if absent. - *

      - * Uses the same sequential access optimization as {@link #getOrCreateColumn(CharSequence, byte, boolean)}. - * When the next expected column is accessed in order, the internal cursor advances without a hash lookup. - */ - public ColumnBuffer getExistingColumn(CharSequence name, byte type) { - return lookupColumn(name, type); - } - /** * Returns the column definitions (cached for efficiency). */ @@ -177,6 +167,16 @@ public QwpColumnDef[] getColumnDefs() { return cachedColumnDefs; } + /** + * Returns an existing column with the given name and type, or {@code null} if absent. + *

      + * Uses the same sequential access optimization as {@link #getOrCreateColumn(CharSequence, byte, boolean)}. + * When the next expected column is accessed in order, the internal cursor advances without a hash lookup. + */ + public ColumnBuffer getExistingColumn(CharSequence name, byte type) { + return lookupColumn(name, type); + } + /** * Gets or creates a column with the given name and type. *

      @@ -192,97 +192,6 @@ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean null return createColumn(name, type, nullable); } - private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) { - ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); - int index = columns.size(); - col.index = index; - columns.add(col); - columnNameToIndex.put(name, index); - // Update fast access array - if (fastColumns == null || index >= fastColumns.length) { - int newLen = Math.max(8, index + 4); - ColumnBuffer[] newArr = new ColumnBuffer[newLen]; - if (fastColumns != null) { - System.arraycopy(fastColumns, 0, newArr, 0, index); - } - fastColumns = newArr; - } - fastColumns[index] = col; - schemaHashComputed = false; - columnDefsCacheValid = false; - return col; - } - - public void rollbackUncommittedColumns() { - if (columns.size() <= committedColumnCount) { - return; - } - - for (int i = columns.size() - 1; i >= committedColumnCount; i--) { - ColumnBuffer col = columns.getQuick(i); - if (col != null) { - col.close(); - } - columns.remove(i); - } - rebuildColumnAccessStructures(); - } - - private void rebuildColumnAccessStructures() { - columnNameToIndex.clear(); - - int columnCount = columns.size(); - int minCapacity = Math.max(8, columnCount + 4); - if (fastColumns == null || fastColumns.length < minCapacity) { - fastColumns = new ColumnBuffer[minCapacity]; - } else { - Arrays.fill(fastColumns, null); - } - - for (int i = 0; i < columnCount; i++) { - ColumnBuffer col = columns.getQuick(i); - col.index = i; - fastColumns[i] = col; - columnNameToIndex.put(col.name, i); - } - - schemaHashComputed = false; - columnDefsCacheValid = false; - cachedColumnDefs = null; - } - - private ColumnBuffer lookupColumn(CharSequence name, byte type) { - // Fast path: predict next column in sequence - int n = columns.size(); - if (columnAccessCursor < n) { - ColumnBuffer candidate = fastColumns[columnAccessCursor]; - if (Chars.equals(candidate.name, name)) { - columnAccessCursor++; - assertColumnType(name, type, candidate); - return candidate; - } - } - - // Slow path: hash map lookup - int idx = columnNameToIndex.get(name); - if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) { - ColumnBuffer existing = columns.get(idx); - assertColumnType(name, type, existing); - return existing; - } - - return null; - } - - private static void assertColumnType(CharSequence name, byte type, ColumnBuffer column) { - if (column.type != type) { - throw new LineSenderException( - "Column type mismatch for column '" + name + "': columnType=" - + column.type + ", sentType=" + type - ); - } - } - /** * Returns the number of rows buffered. */ @@ -290,15 +199,6 @@ public int getRowCount() { return rowCount; } - public boolean hasInProgressRow() { - for (int i = 0, n = columns.size(); i < n; i++) { - if (fastColumns[i].size > rowCount) { - return true; - } - } - return false; - } - /** * Returns the schema hash for this table. *

      @@ -322,6 +222,15 @@ public String getTableName() { return tableName; } + public boolean hasInProgressRow() { + for (int i = 0, n = columns.size(); i < n; i++) { + if (fastColumns[i].size > rowCount) { + return true; + } + } + return false; + } + /** * Advances to the next row. *

      @@ -360,10 +269,21 @@ public void nextRow(ColumnBuffer[] missingColumns, int missingColumnCount) { committedColumnCount = columns.size(); } + /** + * Resets the buffer for reuse. Keeps column definitions and allocated memory. + */ + public void reset() { + for (int i = 0, n = columns.size(); i < n; i++) { + fastColumns[i].reset(); + } + columnAccessCursor = 0; + committedColumnCount = columns.size(); + rowCount = 0; + } + public void retainInProgressRow( int[] sizeBefore, int[] valueCountBefore, - long[] stringDataSizeBefore, int[] arrayShapeOffsetBefore, int[] arrayDataOffsetBefore ) { @@ -374,7 +294,6 @@ public void retainInProgressRow( col.retainTailRow( sizeBefore[i], valueCountBefore[i], - stringDataSizeBefore[i], arrayShapeOffsetBefore[i], arrayDataOffsetBefore[i] ); @@ -386,16 +305,95 @@ public void retainInProgressRow( committedColumnCount = columns.size(); } - /** - * Resets the buffer for reuse. Keeps column definitions and allocated memory. - */ - public void reset() { - for (int i = 0, n = columns.size(); i < n; i++) { - fastColumns[i].reset(); + public void rollbackUncommittedColumns() { + if (columns.size() <= committedColumnCount) { + return; } - columnAccessCursor = 0; - committedColumnCount = columns.size(); - rowCount = 0; + + for (int i = columns.size() - 1; i >= committedColumnCount; i--) { + ColumnBuffer col = columns.getQuick(i); + if (col != null) { + col.close(); + } + columns.remove(i); + } + rebuildColumnAccessStructures(); + } + + private static void assertColumnType(CharSequence name, byte type, ColumnBuffer column) { + if (column.type != type) { + throw new LineSenderException( + "Column type mismatch for column '" + name + "': columnType=" + + column.type + ", sentType=" + type + ); + } + } + + private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) { + ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); + int index = columns.size(); + col.index = index; + columns.add(col); + columnNameToIndex.put(name, index); + // Update fast access array + if (fastColumns == null || index >= fastColumns.length) { + int newLen = Math.max(8, index + 4); + ColumnBuffer[] newArr = new ColumnBuffer[newLen]; + if (fastColumns != null) { + System.arraycopy(fastColumns, 0, newArr, 0, index); + } + fastColumns = newArr; + } + fastColumns[index] = col; + schemaHashComputed = false; + columnDefsCacheValid = false; + return col; + } + + private ColumnBuffer lookupColumn(CharSequence name, byte type) { + // Fast path: predict next column in sequence + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + if (Chars.equals(candidate.name, name)) { + columnAccessCursor++; + assertColumnType(name, type, candidate); + return candidate; + } + } + + // Slow path: hash map lookup + int idx = columnNameToIndex.get(name); + if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + ColumnBuffer existing = columns.get(idx); + assertColumnType(name, type, existing); + return existing; + } + + return null; + } + + private void rebuildColumnAccessStructures() { + columnNameToIndex.clear(); + + int columnCount = columns.size(); + int minCapacity = Math.max(8, columnCount + 4); + if (fastColumns == null || fastColumns.length < minCapacity) { + fastColumns = new ColumnBuffer[minCapacity]; + } else { + Arrays.fill(fastColumns, null); + } + + for (int i = 0; i < columnCount; i++) { + ColumnBuffer col = columns.getQuick(i); + col.index = i; + fastColumns[i] = col; + columnNameToIndex.put(col.name, i); + } + + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; } /** @@ -552,8 +550,8 @@ public static class ColumnBuffer implements QuietCloseable { private OffHeapAppendMemory stringOffsets; // Symbol specific (dictionary stays on-heap) private CharSequenceIntHashMap symbolDict; - private StringSink symbolLookupSink; private ObjList symbolList; + private StringSink symbolLookupSink; private int valueCount; // Actual stored values (excludes nulls) public ColumnBuffer(String name, byte type, boolean nullable) { @@ -786,7 +784,7 @@ public void addDoubleArray(DoubleArray array) { } public void addDoubleArrayPayload(long ptr, long len) { - appendArrayPayload(ptr, len, false); + appendArrayPayload(ptr, len); } public void addFloat(float value) { @@ -939,10 +937,6 @@ public void addLongArray(LongArray array) { size++; } - public void addLongArrayPayload(long ptr, long len) { - appendArrayPayload(ptr, len, true); - } - public void addNull() { if (nullable) { ensureNullCapacity(size + 1); @@ -1086,21 +1080,6 @@ public void addSymbolWithGlobalId(String value, int globalId) { size++; } - public int getIndex() { - return index; - } - - private int getOrAddLocalSymbol(CharSequence value) { - int idx = symbolDict.get(value); - if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - String symbol = Chars.toString(value); - idx = symbolList.size(); - symbolDict.put(symbol, idx); - symbolList.add(symbol); - } - return idx; - } - public void addUuid(long high, long low) { ensureNullBitmapForNonNull(); // Store in wire order: lo first, hi second @@ -1135,14 +1114,14 @@ public void close() { } } - public byte[] getArrayDims() { - return arrayDims; - } - public int getArrayDataOffset() { return arrayDataOffset; } + public byte[] getArrayDims() { + return arrayDims; + } + public int getArrayShapeOffset() { return arrayShapeOffset; } @@ -1204,6 +1183,10 @@ public int getGeoHashPrecision() { return geohashPrecision; } + public int getIndex() { + return index; + } + public long[] getLongArrayData() { return longArrayData; } @@ -1259,22 +1242,18 @@ public CharSequence getSymbolValue(int index) { return symbolList != null ? symbolList.getQuick(index) : null; } - public boolean hasSymbol(CharSequence value) { - return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE; - } - public byte getType() { return type; } - public boolean isNullable() { - return nullable; - } - public int getValueCount() { return valueCount; } + public boolean hasSymbol(CharSequence value) { + return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + } + public boolean isNull(int index) { if (nullBufPtr == 0) { return false; @@ -1284,6 +1263,10 @@ public boolean isNull(int index) { return (Unsafe.getUnsafe().getLong(longAddr) & (1L << bitIndex)) != 0; } + public boolean isNullable() { + return nullable; + } + public void reset() { size = 0; valueCount = 0; @@ -1318,7 +1301,6 @@ public void reset() { public void retainTailRow( int sizeBefore, int valueCountBefore, - long stringDataSizeBefore, int arrayShapeOffsetBefore, int arrayDataOffsetBefore ) { @@ -1430,120 +1412,6 @@ private static int checkedElementCount(long product) { return (int) product; } - private void clearValuePayload() { - if (dataBuffer != null && elemSize > 0) { - dataBuffer.jumpTo(0); - } - if (auxBuffer != null) { - auxBuffer.truncate(); - } - if (stringOffsets != null) { - stringOffsets.truncate(); - stringOffsets.putInt(0); - } - if (stringData != null) { - stringData.truncate(); - } - arrayShapeOffset = 0; - arrayDataOffset = 0; - resetEmptyMetadata(); - } - - private void clearToEmptyFast() { - int sizeBefore = size; - clearValuePayload(); - if (nullBufPtr != 0 && sizeBefore > 0) { - long usedLongs = ((long) sizeBefore + 63) >>> 6; - Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); - } - size = 0; - valueCount = 0; - hasNulls = false; - } - - private void compactNullBitmap(int sourceRow) { - if (nullBufPtr == 0) { - return; - } - - boolean retainedNull = isNull(sourceRow); - long usedLongs = ((long) size + 63) >>> 6; - Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); - if (retainedNull) { - Unsafe.getUnsafe().putLong(nullBufPtr, 1L); - } - hasNulls = retainedNull; - } - - private void appendArrayPayload(long ptr, long len, boolean forLong) { - if (len < 0) { - addNull(); - return; - } - if (len == 0) { - throw new LineSenderException("invalid array payload: empty payload"); - } - - int nDims = Unsafe.getUnsafe().getByte(ptr) & 0xFF; - if (nDims < 1 || nDims > ColumnType.ARRAY_NDIMS_LIMIT) { - throw new LineSenderException("invalid array payload: bad dimensionality " + nDims); - } - - long cursor = ptr + 1; - long headerBytes = 1L + (long) nDims * Integer.BYTES; - if (len < headerBytes) { - throw new LineSenderException("invalid array payload: truncated shape header"); - } - - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - int dimLen = Unsafe.getUnsafe().getInt(cursor); - if (dimLen < 0) { - throw new LineSenderException("invalid array payload: negative dimension length"); - } - elemCount = checkedElementCount((long) elemCount * dimLen); - cursor += Integer.BYTES; - } - - long dataBytes = (long) elemCount * (forLong ? Long.BYTES : Double.BYTES); - if (len != headerBytes + dataBytes) { - throw new LineSenderException("invalid array payload: length mismatch"); - } - - ensureArrayCapacity(nDims, elemCount); - arrayDims[valueCount] = (byte) nDims; - - cursor = ptr + 1; - for (int d = 0; d < nDims; d++) { - arrayShapes[arrayShapeOffset++] = Unsafe.getUnsafe().getInt(cursor); - cursor += Integer.BYTES; - } - - if (dataBytes > 0) { - if (forLong) { - Unsafe.getUnsafe().copyMemory( - null, - cursor, - longArrayData, - LONG_ARRAY_BASE_OFFSET + (long) arrayDataOffset * Long.BYTES, - dataBytes - ); - } else { - Unsafe.getUnsafe().copyMemory( - null, - cursor, - doubleArrayData, - DOUBLE_ARRAY_BASE_OFFSET + (long) arrayDataOffset * Double.BYTES, - dataBytes - ); - } - } - - arrayDataOffset += elemCount; - valueCount++; - size++; - } - private void allocateStorage(byte type) { switch (type) { case TYPE_BOOLEAN: @@ -1600,6 +1468,110 @@ private void allocateStorage(byte type) { } } + private void appendArrayPayload(long ptr, long len) { + if (len < 0) { + addNull(); + return; + } + if (len == 0) { + throw new LineSenderException("invalid array payload: empty payload"); + } + + int nDims = Unsafe.getUnsafe().getByte(ptr) & 0xFF; + if (nDims < 1 || nDims > ColumnType.ARRAY_NDIMS_LIMIT) { + throw new LineSenderException("invalid array payload: bad dimensionality " + nDims); + } + + long cursor = ptr + 1; + long headerBytes = 1L + (long) nDims * Integer.BYTES; + if (len < headerBytes) { + throw new LineSenderException("invalid array payload: truncated shape header"); + } + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = Unsafe.getUnsafe().getInt(cursor); + if (dimLen < 0) { + throw new LineSenderException("invalid array payload: negative dimension length"); + } + elemCount = checkedElementCount((long) elemCount * dimLen); + cursor += Integer.BYTES; + } + + long dataBytes = (long) elemCount * Double.BYTES; + if (len != headerBytes + dataBytes) { + throw new LineSenderException("invalid array payload: length mismatch"); + } + + ensureArrayCapacity(nDims, elemCount); + arrayDims[valueCount] = (byte) nDims; + + cursor = ptr + 1; + for (int d = 0; d < nDims; d++) { + arrayShapes[arrayShapeOffset++] = Unsafe.getUnsafe().getInt(cursor); + cursor += Integer.BYTES; + } + + if (dataBytes > 0) { + Unsafe.getUnsafe().copyMemory( + null, + cursor, + doubleArrayData, + DOUBLE_ARRAY_BASE_OFFSET + (long) arrayDataOffset * Double.BYTES, + dataBytes + ); + } + + arrayDataOffset += elemCount; + valueCount++; + size++; + } + + private void clearToEmptyFast() { + int sizeBefore = size; + clearValuePayload(); + if (nullBufPtr != 0 && sizeBefore > 0) { + long usedLongs = ((long) sizeBefore + 63) >>> 6; + Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); + } + size = 0; + valueCount = 0; + hasNulls = false; + } + + private void clearValuePayload() { + if (dataBuffer != null && elemSize > 0) { + dataBuffer.jumpTo(0); + } + if (auxBuffer != null) { + auxBuffer.truncate(); + } + if (stringOffsets != null) { + stringOffsets.truncate(); + stringOffsets.putInt(0); + } + if (stringData != null) { + stringData.truncate(); + } + arrayShapeOffset = 0; + arrayDataOffset = 0; + resetEmptyMetadata(); + } + + private void compactNullBitmap(int sourceRow) { + if (nullBufPtr == 0) { + return; + } + + boolean retainedNull = isNull(sourceRow); + long usedLongs = ((long) size + 63) >>> 6; + Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); + if (retainedNull) { + Unsafe.getUnsafe().putLong(nullBufPtr, 1L); + } + hasNulls = retainedNull; + } + private void ensureArrayCapacity(int nDims, int dataElements) { // Ensure per-row array dims capacity if (valueCount >= arrayDims.length) { @@ -1653,6 +1625,17 @@ private void ensureNullCapacity(int rows) { } } + private int getOrAddLocalSymbol(CharSequence value) { + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + String symbol = Chars.toString(value); + idx = symbolList.size(); + symbolDict.put(symbol, idx); + symbolList.add(symbol); + } + return idx; + } + private void markNull(int index) { long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; int bitIndex = index & 63; @@ -1661,6 +1644,16 @@ private void markNull(int index) { hasNulls = true; } + private void resetEmptyMetadata() { + decimalScale = -1; + geohashPrecision = -1; + maxGlobalSymbolId = -1; + if (symbolDict != null) { + symbolDict.clear(); + symbolList.clear(); + } + } + private void retainArrayValue(int valueIndex, int shapeOffsetBefore, int dataOffsetBefore) { int nDims = arrayDims[valueIndex] & 0xFF; arrayDims[0] = (byte) nDims; @@ -1733,15 +1726,5 @@ private void retainSymbolValue(int valueIndex) { symbolDict.put(symbol, 0); Unsafe.getUnsafe().putInt(dataBuffer.pageAddress(), 0); } - - private void resetEmptyMetadata() { - decimalScale = -1; - geohashPrecision = -1; - maxGlobalSymbolId = -1; - if (symbolDict != null) { - symbolDict.clear(); - symbolList.clear(); - } - } } } From d67c81eb45a3e7b42f778b48bee10f9f1bd82660 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 10 Mar 2026 12:21:49 +0100 Subject: [PATCH 175/230] Make all column types >= 32 bits use null bitmap --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 00aea36..85164f3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -639,7 +639,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_DOUBLE, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_DOUBLE, true); col.addDouble(value); return this; } @@ -654,7 +654,7 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_FLOAT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_FLOAT, true); col.addFloat(value); return this; } @@ -749,7 +749,7 @@ public QwpTableBuffer getTableBuffer(String tableName) { public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_INT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_INT, true); col.addInt(value); return this; } @@ -823,7 +823,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG, true); col.addLong(value); return this; } From 8e55257a9a025f7dec9ea134dc57b1cf2a6206e6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 10 Mar 2026 12:22:00 +0100 Subject: [PATCH 176/230] Lint --- .../questdb/client/cutlass/qwp/client/QwpWebSocketSender.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 85164f3..0975140 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -738,7 +738,6 @@ public QwpTableBuffer getTableBuffer(String tableName) { return buffer; } - /** * Adds an INT column value to the current row. * @@ -996,7 +995,7 @@ private void checkTableSelected() { private String checkedColumnName(CharSequence name) { if (name == null || !TableUtils.isValidColumnName(name, DEFAULT_MAX_NAME_LENGTH)) { - if (name == null || name.length() == 0) { + if (name == null || name.isEmpty()) { throw new LineSenderException("column name cannot be empty"); } if (name.length() > DEFAULT_MAX_NAME_LENGTH) { From 77e1259b935623c0bf3f7c632ae04725c8a9f732 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 10 Mar 2026 12:22:05 +0100 Subject: [PATCH 177/230] Javadoc --- .../cutlass/qwp/client/QwpWebSocketSender.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 0975140..7bc0e3e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -85,12 +85,12 @@ * } * *

      - * Fast-path API for high-throughput generators + *

      Fast-path API for high-throughput generators

      *

      * For maximum throughput, bypass the fluent API to avoid per-row overhead * (no column-name hashmap lookups, no {@code checkNotClosed()}/{@code checkTableSelected()} - * per column, direct access to column buffers). Use {@link #getTableBuffer(String)}, - * {@link #getOrAddGlobalSymbol(String)}, and {@link #incrementPendingRowCount()}: + * per column, direct access to column buffers). The entry point is + * {@link #getTableBuffer(String)}. * 

        * // Setup (once)
  * QwpTableBuffer tableBuffer = sender.getTableBuffer("q");
@@ -724,8 +724,10 @@ public int getPendingRowCount() {
     }
 
     /**
-     * Gets or creates a table buffer for direct access.
-     * For high-throughput generators that want to bypass fluent API overhead.
+     * DANGER:: gets or creates a low-level table buffer,
+     * allowing you to bypass some validation and enforcement of invariants
+     * present in the higher-level API. When used incorrectly, it may result
+     * in silent data loss.
      */
     public QwpTableBuffer getTableBuffer(String tableName) {
         QwpTableBuffer buffer = tableBuffers.get(tableName);

From 78c9f225fc249f76678ec496e95b51ef4e8fbdc4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 10 Mar 2026 15:37:23 +0100
Subject: [PATCH 178/230] Auto-reorganize code

---
 .../cutlass/qwp/client/QwpUdpSenderTest.java  | 2056 ++++++++---------
 1 file changed, 1020 insertions(+), 1036 deletions(-)

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
index b663a10..2bd95fb 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
@@ -50,103 +50,11 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.SCHEMA_MODE_FULL;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_BOOLEAN;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DECIMAL128;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DECIMAL256;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DECIMAL64;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DOUBLE;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_DOUBLE_ARRAY;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_LONG;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_LONG_ARRAY;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_STRING;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_SYMBOL;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_TIMESTAMP;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_TIMESTAMP_NANOS;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_VARCHAR;
-import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.VERSION_1;
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
 import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
 
 public class QwpUdpSenderTest {
 
-    @Test
-    public void testFirstRowAllowsMultipleNewColumnsAndEncodesRow() throws Exception {
-        assertMemoryLeak(() -> {
-            List rows = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("a", 1)
-                                    .doubleColumn("b", 2.0)
-                                    .stringColumn("c", "x")
-                                    .atNow(),
-                            "a", 1L,
-                            "b", 2.0,
-                            "c", "x")
-            );
-
-            RunResult result = runScenario(rows, 1024 * 1024);
-            Assert.assertEquals(1, result.sendCount);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
-        });
-    }
-
-    @Test
-    public void testBoundedSenderMixedNullablePaddingPreservesRowsAndPacketLimit() throws Exception {
-        assertMemoryLeak(() -> {
-            String alpha = repeat('a', 256);
-            String omega = repeat('z', 256);
-            List rows = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .symbol("sym", "v1")
-                                    .longColumn("x", 1)
-                                    .stringColumn("s", alpha)
-                                    .atNow(),
-                            "sym", "v1",
-                            "x", 1L,
-                            "s", alpha),
-                    row("t", sender -> sender.table("t")
-                                    .symbol("sym", null)
-                                    .longColumn("x", 2)
-                                    .atNow(),
-                            "sym", null,
-                            "x", 2L,
-                            "s", null),
-                    row("t", sender -> sender.table("t")
-                                    .symbol("sym", null)
-                                    .longColumn("x", 3)
-                                    .stringColumn("s", null)
-                                    .atNow(),
-                            "sym", null,
-                            "x", 3L,
-                            "s", null),
-                    row("t", sender -> sender.table("t")
-                                    .symbol("sym", "v2")
-                                    .longColumn("x", 4)
-                                    .stringColumn("s", omega)
-                                    .atNow(),
-                            "sym", "v2",
-                            "x", 4L,
-                            "s", omega)
-            );
-
-            int maxDatagramSize = fullPacketSize(rows) - 1;
-            RunResult result = runScenario(rows, maxDatagramSize);
-
-            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
-            assertPacketsWithinLimit(result, maxDatagramSize);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
-        });
-    }
-
-    @Test
-    public void testEstimateMatchesActualEncodedSize() throws Exception {
-        assertMemoryLeak(() -> {
-            auditEstimateWithStableSchemaAndNullableValues();
-            auditEstimateAcrossSymbolDictionaryVarintBoundary();
-        });
-    }
-
     @Test
     public void testAdaptiveHeadroomFlushesCommittedRowsBeforeNextRowStarts() throws Exception {
         assertMemoryLeak(() -> {
@@ -280,165 +188,161 @@ public void testAdaptiveHeadroomStateIsPerTable() throws Exception {
     }
 
     @Test
-    public void testBoundedSenderNullableStringNullAcrossOverflowBoundaryPreservesRowsAndPacketLimit() throws Exception {
+    public void testArrayWrapperStagingSnapshotsMutationAndCancelRow() throws Exception {
         assertMemoryLeak(() -> {
-            String alpha = repeat('a', 512);
-            String marker1 = repeat('m', 64);
-            String marker2 = repeat('n', 64);
-            String marker3 = repeat('o', 64);
-            String omega = repeat('z', 128);
-            List rows = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("x", 1)
-                                    .stringColumn("s", alpha)
-                                    .stringColumn("m", marker1)
-                                    .atNow(),
-                            "x", 1L,
-                            "s", alpha,
-                            "m", marker1),
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("x", 2)
-                                    .stringColumn("s", null)
-                                    .stringColumn("m", marker2)
-                                    .atNow(),
-                            "x", 2L,
-                            "s", null,
-                            "m", marker2),
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("x", 3)
-                                    .stringColumn("s", omega)
-                                    .stringColumn("m", marker3)
-                                    .atNow(),
-                            "x", 3L,
-                            "s", omega,
-                            "m", marker3)
-            );
-            int firstRowPacket = fullPacketSize(rows.subList(0, 1));
-            int twoRowPacket = fullPacketSize(rows.subList(0, 2));
-            int maxDatagramSize = firstRowPacket + 16;
-            Assert.assertTrue("expected overflow boundary between row 1 and row 2", maxDatagramSize < twoRowPacket);
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024);
+                 DoubleArray doubleArray = new DoubleArray(2)) {
+                sender.table("arrays");
+                long[] longValues = {1, 2};
 
-            RunResult result = runScenario(rows, maxDatagramSize);
+                doubleArray.append(1.5).append(2.5);
+                sender.longArray("la", longValues);
+                sender.doubleArray("da", doubleArray);
 
-            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
-            assertPacketsWithinLimit(result, maxDatagramSize);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+                longValues[0] = 9;
+                longValues[1] = 10;
+                doubleArray.clear();
+                doubleArray.append(9.5).append(10.5);
+                sender.cancelRow();
+
+                sender.longArray("la", longValues);
+                sender.doubleArray("da", doubleArray);
+                longValues[0] = 100;
+                longValues[1] = 200;
+                doubleArray.clear();
+                doubleArray.append(100.5).append(200.5);
+                sender.atNow();
+                sender.flush();
+            }
+
+            assertRowsEqual(
+                    Arrays.asList(decodedRow(
+                            "arrays",
+                            "la", longArrayValue(shape(2), 9, 10),
+                            "da", doubleArrayValue(shape(2), 9.5, 10.5)
+                    )),
+                    decodeRows(nf.packets)
+            );
         });
     }
 
     @Test
-    public void testUnboundedSenderOmittedNullableAndNonNullableColumnsPreservesRows() throws Exception {
+    public void testAtMicrosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
         assertMemoryLeak(() -> {
-            List rows = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("x", 1)
-                                    .stringColumn("s", "alpha")
-                                    .symbol("sym", "one")
-                                    .atNow(),
-                            "x", 1L,
-                            "s", "alpha",
-                            "sym", "one"),
-                    row("t", sender -> sender.table("t")
-                                    .stringColumn("s", "beta")
-                                    .atNow(),
-                            "x", Long.MIN_VALUE,
-                            "s", "beta",
-                            "sym", null),
+            String large = repeat('x', 5000);
+            List oversizedRow = Arrays.asList(
                     row("t", sender -> sender.table("t")
-                                    .longColumn("x", 3)
-                                    .atNow(),
-                            "x", 3L,
-                            "s", null,
-                            "sym", null)
+                                    .longColumn("a", 2)
+                                    .stringColumn("s", large)
+                                    .at(2_000_000L, ChronoUnit.MICROS),
+                            "a", 2L,
+                            "s", large,
+                            "", 2_000_000L)
             );
+            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
 
-            RunResult result = runScenario(rows, 0);
-
-            Assert.assertEquals(1, result.sendCount);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
-        });
-    }
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .at(1_000_000L, ChronoUnit.MICROS);
 
-    @Test
-    public void testUnboundedSenderWideSchemaWithLowIndexWritePreservesRows() throws Exception {
-        assertMemoryLeak(() -> {
-            List rows = Arrays.asList(
-                    row("wide", sender -> sender.table("wide")
-                                    .longColumn("c0", 0)
-                                    .longColumn("c1", 1)
-                                    .longColumn("c2", 2)
-                                    .longColumn("c3", 3)
-                                    .longColumn("c4", 4)
-                                    .longColumn("c5", 5)
-                                    .longColumn("c6", 6)
-                                    .longColumn("c7", 7)
-                                    .longColumn("c8", 8)
-                                    .longColumn("c9", 9)
-                                    .atNow(),
-                            "c0", 0L,
-                            "c1", 1L,
-                            "c2", 2L,
-                            "c3", 3L,
-                            "c4", 4L,
-                            "c5", 5L,
-                            "c6", 6L,
-                            "c7", 7L,
-                            "c8", 8L,
-                            "c9", 9L),
-                    row("wide", sender -> sender.table("wide")
-                                    .longColumn("c0", 10)
-                                    .atNow(),
-                            "c0", 10L,
-                            "c1", Long.MIN_VALUE,
-                            "c2", Long.MIN_VALUE,
-                            "c3", Long.MIN_VALUE,
-                            "c4", Long.MIN_VALUE,
-                            "c5", Long.MIN_VALUE,
-                            "c6", Long.MIN_VALUE,
-                            "c7", Long.MIN_VALUE,
-                            "c8", Long.MIN_VALUE,
-                            "c9", Long.MIN_VALUE)
-            );
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("a", 2)
+                                .stringColumn("s", large)
+                                .at(2_000_000L, ChronoUnit.MICROS)
+                );
+                Assert.assertEquals(1, nf.sendCount);
 
-            RunResult result = runScenario(rows, 0);
+                sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.MICROS);
+                sender.flush();
+            }
 
-            Assert.assertEquals(1, result.sendCount);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L, "", 1_000_000L),
+                    decodedRow("t", "a", 3L, "", 3_000_000L)
+            ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testBoundedSenderOmittedNonNullableColumnsPreservesRowsAndPacketLimit() throws Exception {
+    public void testAtNanosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
         assertMemoryLeak(() -> {
-            String alpha = repeat('a', 256);
-            String beta = repeat('b', 192);
-            String omega = repeat('z', 256);
-            List rows = Arrays.asList(
-                    row("mix", sender -> sender.table("mix")
-                                    .longColumn("x", 1)
-                                    .stringColumn("msg", alpha)
-                                    .atNow(),
-                            "x", 1L,
-                            "msg", alpha),
-                    row("mix", sender -> sender.table("mix")
-                                    .stringColumn("msg", beta)
-                                    .atNow(),
-                            "x", Long.MIN_VALUE,
-                            "msg", beta),
-                    row("mix", sender -> sender.table("mix")
-                                    .longColumn("x", 3)
-                                    .stringColumn("msg", omega)
+            String large = repeat('x', 5000);
+            List oversizedRow = Arrays.asList(
+                    row("tn", sender -> sender.table("tn")
+                                    .longColumn("a", 2)
+                                    .stringColumn("s", large)
+                                    .at(2_000_000L, ChronoUnit.NANOS),
+                            "a", 2L,
+                            "s", large,
+                            "", 2_000_000L)
+            );
+            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
+
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("tn")
+                        .longColumn("a", 1)
+                        .at(1_000_000L, ChronoUnit.NANOS);
+
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("a", 2)
+                                .stringColumn("s", large)
+                                .at(2_000_000L, ChronoUnit.NANOS)
+                );
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.NANOS);
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("tn", "a", 1L, "", 1_000_000L),
+                    decodedRow("tn", "a", 3L, "", 3_000_000L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testAtNowOversizeFailureRollsBackWithoutExplicitCancel() throws Exception {
+        assertMemoryLeak(() -> {
+            String large = repeat('x', 5000);
+            List oversizedRow = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("a", 2)
+                                    .stringColumn("s", large)
                                     .atNow(),
-                            "x", 3L,
-                            "msg", omega)
+                            "a", 2L,
+                            "s", large)
             );
+            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
 
-            int maxDatagramSize = fullPacketSize(rows) - 1;
-            RunResult result = runScenario(rows, maxDatagramSize);
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .atNow();
 
-            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
-            assertPacketsWithinLimit(result, maxDatagramSize);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("a", 2)
+                                .stringColumn("s", large)
+                                .atNow()
+                );
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.longColumn("a", 3).atNow();
+                sender.flush();
+            }
+
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L),
+                    decodedRow("t", "a", 3L)
+            ), decodeRows(nf.packets));
         });
     }
 
@@ -545,69 +449,50 @@ public void testBoundedSenderHigherDimensionalDoubleArrayWrapperPreservesRowsAnd
     }
 
     @Test
-    public void testArrayWrapperStagingSnapshotsMutationAndCancelRow() throws Exception {
+    public void testBoundedSenderMixedNullablePaddingPreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024);
-                 DoubleArray doubleArray = new DoubleArray(2)) {
-                sender.table("arrays");
-                long[] longValues = {1, 2};
-
-                doubleArray.append(1.5).append(2.5);
-                sender.longArray("la", longValues);
-                sender.doubleArray("da", doubleArray);
-
-                longValues[0] = 9;
-                longValues[1] = 10;
-                doubleArray.clear();
-                doubleArray.append(9.5).append(10.5);
-                sender.cancelRow();
-
-                sender.longArray("la", longValues);
-                sender.doubleArray("da", doubleArray);
-                longValues[0] = 100;
-                longValues[1] = 200;
-                doubleArray.clear();
-                doubleArray.append(100.5).append(200.5);
-                sender.atNow();
-                sender.flush();
-            }
-
-            assertRowsEqual(
-                    Arrays.asList(decodedRow(
-                            "arrays",
-                            "la", longArrayValue(shape(2), 9, 10),
-                            "da", doubleArrayValue(shape(2), 9.5, 10.5)
-                    )),
-                    decodeRows(nf.packets)
+            String alpha = repeat('a', 256);
+            String omega = repeat('z', 256);
+            List rows = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v1")
+                                    .longColumn("x", 1)
+                                    .stringColumn("s", alpha)
+                                    .atNow(),
+                            "sym", "v1",
+                            "x", 1L,
+                            "s", alpha),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", null)
+                                    .longColumn("x", 2)
+                                    .atNow(),
+                            "sym", null,
+                            "x", 2L,
+                            "s", null),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", null)
+                                    .longColumn("x", 3)
+                                    .stringColumn("s", null)
+                                    .atNow(),
+                            "sym", null,
+                            "x", 3L,
+                            "s", null),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v2")
+                                    .longColumn("x", 4)
+                                    .stringColumn("s", omega)
+                                    .atNow(),
+                            "sym", "v2",
+                            "x", 4L,
+                            "s", omega)
             );
-        });
-    }
-
-    @Test
-    public void testLongArrayWrapperStagingSnapshotsMutation() throws Exception {
-        assertMemoryLeak(() -> {
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024);
-                 LongArray longArray = new LongArray(2, 2)) {
-                sender.table("arrays");
-
-                longArray.append(1).append(2).append(3).append(4);
-                sender.longArray("la", longArray);
 
-                longArray.clear();
-                longArray.append(10).append(20).append(30).append(40);
-                sender.atNow();
-                sender.flush();
-            }
+            int maxDatagramSize = fullPacketSize(rows) - 1;
+            RunResult result = runScenario(rows, maxDatagramSize);
 
-            assertRowsEqual(
-                    Arrays.asList(decodedRow(
-                            "arrays",
-                            "la", longArrayValue(shape(2, 2), 1, 2, 3, 4)
-                    )),
-                    decodeRows(nf.packets)
-            );
+            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
+            assertPacketsWithinLimit(result, maxDatagramSize);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
         });
     }
 
@@ -675,72 +560,98 @@ public void testBoundedSenderMixedTypesPreservesRowsAndPacketLimit() throws Exce
     }
 
     @Test
-    public void testBoundedSenderRepeatedOverflowBoundariesWithDistinctSymbolsPreserveRowsAndPacketLimit() throws Exception {
+    public void testBoundedSenderNullableStringNullAcrossOverflowBoundaryPreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
-            String payload = repeat('p', 256);
+            String alpha = repeat('a', 512);
+            String marker1 = repeat('m', 64);
+            String marker2 = repeat('n', 64);
+            String marker3 = repeat('o', 64);
+            String omega = repeat('z', 128);
             List rows = Arrays.asList(
                     row("t", sender -> sender.table("t")
-                                    .symbol("sym", "v0")
-                                    .longColumn("x", 0)
-                                    .stringColumn("s", payload)
-                                    .atNow(),
-                            "sym", "v0",
-                            "x", 0L,
-                            "s", payload),
-                    row("t", sender -> sender.table("t")
-                                    .symbol("sym", "v1")
                                     .longColumn("x", 1)
-                                    .stringColumn("s", payload)
+                                    .stringColumn("s", alpha)
+                                    .stringColumn("m", marker1)
                                     .atNow(),
-                            "sym", "v1",
                             "x", 1L,
-                            "s", payload),
+                            "s", alpha,
+                            "m", marker1),
                     row("t", sender -> sender.table("t")
-                                    .symbol("sym", "v2")
                                     .longColumn("x", 2)
-                                    .stringColumn("s", payload)
+                                    .stringColumn("s", null)
+                                    .stringColumn("m", marker2)
                                     .atNow(),
-                            "sym", "v2",
                             "x", 2L,
-                            "s", payload),
+                            "s", null,
+                            "m", marker2),
                     row("t", sender -> sender.table("t")
-                                    .symbol("sym", "v3")
                                     .longColumn("x", 3)
-                                    .stringColumn("s", payload)
+                                    .stringColumn("s", omega)
+                                    .stringColumn("m", marker3)
                                     .atNow(),
-                            "sym", "v3",
                             "x", 3L,
-                            "s", payload),
-                    row("t", sender -> sender.table("t")
-                                    .symbol("sym", "v4")
-                                    .longColumn("x", 4)
-                                    .stringColumn("s", payload)
-                                    .atNow(),
-                            "sym", "v4",
-                            "x", 4L,
-                            "s", payload)
+                            "s", omega,
+                            "m", marker3)
             );
-            int maxDatagramSize = fullPacketSize(rows.subList(0, 2)) - 1;
+            int firstRowPacket = fullPacketSize(rows.subList(0, 1));
+            int twoRowPacket = fullPacketSize(rows.subList(0, 2));
+            int maxDatagramSize = firstRowPacket + 16;
+            Assert.assertTrue("expected overflow boundary between row 1 and row 2", maxDatagramSize < twoRowPacket);
 
             RunResult result = runScenario(rows, maxDatagramSize);
 
-            Assert.assertEquals(rows.size(), result.sendCount);
+            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
             assertPacketsWithinLimit(result, maxDatagramSize);
             assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
         });
     }
 
     @Test
-    public void testBoundedSenderOutOfOrderExistingColumnsPreservesRowsAndPacketLimit() throws Exception {
+    public void testBoundedSenderOmittedNonNullableColumnsPreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
+            String alpha = repeat('a', 256);
+            String beta = repeat('b', 192);
+            String omega = repeat('z', 256);
             List rows = Arrays.asList(
-                    row("order", sender -> sender.table("order")
-                                    .longColumn("a", 1)
-                                    .stringColumn("b", "x")
-                                    .symbol("c", "alpha")
+                    row("mix", sender -> sender.table("mix")
+                                    .longColumn("x", 1)
+                                    .stringColumn("msg", alpha)
                                     .atNow(),
-                            "a", 1L,
-                            "b", "x",
+                            "x", 1L,
+                            "msg", alpha),
+                    row("mix", sender -> sender.table("mix")
+                                    .stringColumn("msg", beta)
+                                    .atNow(),
+                            "x", Long.MIN_VALUE,
+                            "msg", beta),
+                    row("mix", sender -> sender.table("mix")
+                                    .longColumn("x", 3)
+                                    .stringColumn("msg", omega)
+                                    .atNow(),
+                            "x", 3L,
+                            "msg", omega)
+            );
+
+            int maxDatagramSize = fullPacketSize(rows) - 1;
+            RunResult result = runScenario(rows, maxDatagramSize);
+
+            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
+            assertPacketsWithinLimit(result, maxDatagramSize);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
+    @Test
+    public void testBoundedSenderOutOfOrderExistingColumnsPreservesRowsAndPacketLimit() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("order", sender -> sender.table("order")
+                                    .longColumn("a", 1)
+                                    .stringColumn("b", "x")
+                                    .symbol("c", "alpha")
+                                    .atNow(),
+                            "a", 1L,
+                            "b", "x",
                             "c", "alpha"),
                     row("order", sender -> sender.table("order")
                                     .symbol("c", "beta")
@@ -770,44 +681,188 @@ public void testBoundedSenderOutOfOrderExistingColumnsPreservesRowsAndPacketLimi
     }
 
     @Test
-    public void testMixingAtNowAndAtMicrosAfterCommittedRowsSplitsDatagramAndPreservesRows() throws Exception {
+    public void testBoundedSenderRepeatedOverflowBoundariesWithDistinctSymbolsPreserveRowsAndPacketLimit() throws Exception {
+        assertMemoryLeak(() -> {
+            String payload = repeat('p', 256);
+            List rows = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v0")
+                                    .longColumn("x", 0)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v0",
+                            "x", 0L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v1")
+                                    .longColumn("x", 1)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v1",
+                            "x", 1L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v2")
+                                    .longColumn("x", 2)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v2",
+                            "x", 2L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v3")
+                                    .longColumn("x", 3)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v3",
+                            "x", 3L,
+                            "s", payload),
+                    row("t", sender -> sender.table("t")
+                                    .symbol("sym", "v4")
+                                    .longColumn("x", 4)
+                                    .stringColumn("s", payload)
+                                    .atNow(),
+                            "sym", "v4",
+                            "x", 4L,
+                            "s", payload)
+            );
+            int maxDatagramSize = fullPacketSize(rows.subList(0, 2)) - 1;
+
+            RunResult result = runScenario(rows, maxDatagramSize);
+
+            Assert.assertEquals(rows.size(), result.sendCount);
+            assertPacketsWithinLimit(result, maxDatagramSize);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
+    @Test
+    public void testBoundedSenderSchemaFlushThenOmittedNullableColumnsPreservesRows() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("schema", sender -> sender.table("schema")
+                                    .longColumn("x", 1)
+                                    .stringColumn("s", "alpha")
+                                    .atNow(),
+                            "x", 1L,
+                            "s", "alpha"),
+                    row("schema", sender -> sender.table("schema")
+                                    .symbol("sym", "new")
+                                    .longColumn("x", 2)
+                                    .stringColumn("s", "beta")
+                                    .atNow(),
+                            "sym", "new",
+                            "x", 2L,
+                            "s", "beta"),
+                    row("schema", sender -> sender.table("schema")
+                                    .longColumn("x", 3)
+                                    .atNow(),
+                            "sym", null,
+                            "x", 3L,
+                            "s", null)
+            );
+
+            RunResult result = runScenario(rows, 1024 * 1024);
+
+            Assert.assertEquals(2, result.sendCount);
+            assertPacketsWithinLimit(result, 1024 * 1024);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
+    @Test
+    public void testCancelRowAfterMidRowSchemaChangeDoesNotLeakSchema() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
             try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
-                        .longColumn("x", 1)
+                        .longColumn("a", 1)
                         .atNow();
 
-                sender.longColumn("x", 2);
-                sender.at(2, ChronoUnit.MICROS);
+                sender.longColumn("a", 2);
+                sender.longColumn("b", 3);
                 Assert.assertEquals(1, nf.sendCount);
 
+                sender.cancelRow();
+                sender.longColumn("a", 4).atNow();
                 sender.flush();
             }
 
             Assert.assertEquals(2, nf.sendCount);
             assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "x", 1L),
-                    decodedRow("t", "x", 2L, "", 2L)
+                    decodedRow("t", "a", 1L),
+                    decodedRow("t", "a", 4L)
             ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testFlushWhileRowInProgressThrowsAndPreservesRow() throws Exception {
+    public void testCloseDropsInProgressRowButFlushesCommittedRows() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
             try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
-                sender.table("t").longColumn("x", 1);
+                sender.table("t").longColumn("x", 1).atNow();
+                sender.longColumn("x", 2);
+            }
 
-                assertThrowsContains("Cannot flush buffer while row is in progress", sender::flush);
+            Assert.assertEquals(1, nf.sendCount);
+            assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
+        });
+    }
 
-                sender.atNow();
+    @Test
+    public void testDuplicateColumnAfterSchemaFlushReplayIsRejected() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longColumn("a", 1)
+                        .atNow();
+
+                sender.longColumn("a", 2);
+                sender.longColumn("b", 3);
+                Assert.assertEquals(1, nf.sendCount);
+
+                assertThrowsContains("column 'a' already set for current row", () -> sender.longColumn("a", 4));
+
+                sender.cancelRow();
+                sender.longColumn("a", 5).atNow();
                 sender.flush();
             }
 
-            Assert.assertEquals(1, nf.sendCount);
-            assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "a", 1L),
+                    decodedRow("t", "a", 5L)
+            ), decodeRows(nf.packets));
+        });
+    }
+
+    @Test
+    public void testEstimateMatchesActualEncodedSize() throws Exception {
+        assertMemoryLeak(() -> {
+            auditEstimateWithStableSchemaAndNullableValues();
+            auditEstimateAcrossSymbolDictionaryVarintBoundary();
+        });
+    }
+
+    @Test
+    public void testFirstRowAllowsMultipleNewColumnsAndEncodesRow() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("a", 1)
+                                    .doubleColumn("b", 2.0)
+                                    .stringColumn("c", "x")
+                                    .atNow(),
+                            "a", 1L,
+                            "b", 2.0,
+                            "c", "x")
+            );
+
+            RunResult result = runScenario(rows, 1024 * 1024);
+            Assert.assertEquals(1, result.sendCount);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
         });
     }
 
@@ -836,122 +891,191 @@ public void testFlushPreservesPendingFillStateForCurrentTable() throws Exception
     }
 
     @Test
-    public void testSimpleLongRowUsesScatterSendPath() throws Exception {
+    public void testFlushWhileRowInProgressThrowsAndPreservesRow() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
             try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
-                sender.table("t").longColumn("x", 42L).atNow();
+                sender.table("t").longColumn("x", 1);
+
+                assertThrowsContains("Cannot flush buffer while row is in progress", sender::flush);
+
+                sender.atNow();
                 sender.flush();
             }
 
             Assert.assertEquals(1, nf.sendCount);
-            Assert.assertEquals(1, nf.scatterSendCount);
-            Assert.assertEquals(0, nf.rawSendCount);
-            Assert.assertTrue("expected multiple segments for header/schema/data", nf.segmentCounts.get(0) > 1);
-            assertRowsEqual(Arrays.asList(decodedRow("t", "x", 42L)), decodeRows(nf.packets));
+            assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testSwitchTableWhileRowInProgressThrowsAndPreservesRows() throws Exception {
+    public void testIrregularArrayRejectedDuringStagingAndSenderRemainsUsable() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
-                sender.table("t1").longColumn("x", 1);
-
-                assertThrowsContains("Cannot switch tables while row is in progress",
-                        () -> sender.table("t2"));
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("arrays");
+                assertThrowsContains("irregular array shape", () ->
+                        sender.doubleArray("da", new double[][]{{1.0}, {2.0, 3.0}})
+                );
 
-                sender.atNow();
-                sender.table("t2").longColumn("y", 2).atNow();
+                sender.table("ok");
+                sender.longColumn("x", 1).atNow();
                 sender.flush();
             }
 
-            Assert.assertEquals(2, nf.sendCount);
-            assertRowsEqualIgnoringOrder(
-                    Arrays.asList(decodedRow("t1", "x", 1L), decodedRow("t2", "y", 2L)),
-                    decodeRows(nf.packets)
-            );
+            Assert.assertEquals(1, nf.sendCount);
+            assertRowsEqual(Arrays.asList(decodedRow("ok", "x", 1L)), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testCloseDropsInProgressRowButFlushesCommittedRows() throws Exception {
+    public void testIrregularArrayRejectedDuringStagingDoesNotLeakColumnIntoSameTable() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
-                sender.table("t").longColumn("x", 1).atNow();
-                sender.longColumn("x", 2);
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("arrays");
+                assertThrowsContains("irregular array shape", () ->
+                        sender.doubleArray("bad", new double[][]{{1.0}, {2.0, 3.0}})
+                );
+
+                sender.longColumn("x", 1).atNow();
+                sender.flush();
             }
 
             Assert.assertEquals(1, nf.sendCount);
-            assertRowsEqual(Arrays.asList(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
+            assertRowsEqual(Arrays.asList(decodedRow("arrays", "x", 1L)), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testOversizedSingleRowRejectedAfterReplayUsesActualEncodedSize() throws Exception {
+    public void testLongArrayWrapperStagingSnapshotsMutation() throws Exception {
         assertMemoryLeak(() -> {
-            String small = repeat('s', 32);
-            String large = repeat('x', 5000);
-            List largeRow = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("x", 2)
-                                    .stringColumn("s", large)
-                                    .atNow(),
-                            "x", 2L,
-                            "s", large)
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024);
+                 LongArray longArray = new LongArray(2, 2)) {
+                sender.table("arrays");
+
+                longArray.append(1).append(2).append(3).append(4);
+                sender.longArray("la", longArray);
+
+                longArray.clear();
+                longArray.append(10).append(20).append(30).append(40);
+                sender.atNow();
+                sender.flush();
+            }
+
+            assertRowsEqual(
+                    Arrays.asList(decodedRow(
+                            "arrays",
+                            "la", longArrayValue(shape(2, 2), 1, 2, 3, 4)
+                    )),
+                    decodeRows(nf.packets)
             );
-            int maxDatagramSize = fullPacketSize(largeRow) - 1;
+        });
+    }
 
+    @Test
+    public void testMixingAtNowAndAtMicrosAfterCommittedRowsSplitsDatagramAndPreservesRows() throws Exception {
+        assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
                         .longColumn("x", 1)
-                        .stringColumn("s", small)
                         .atNow();
 
-                assertThrowsContains("single row exceeds maximum datagram size", () ->
-                        sender.longColumn("x", 2)
-                                .stringColumn("s", large)
-                                .atNow()
-                );
+                sender.longColumn("x", 2);
+                sender.at(2, ChronoUnit.MICROS);
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.flush();
             }
 
-            Assert.assertEquals(1, nf.sendCount);
-            assertPacketsWithinLimit(new RunResult(nf.packets, nf.lengths, nf.sendCount), maxDatagramSize);
-            assertRowsEqual(
-                    Arrays.asList(decodedRow("t", "x", 1L, "s", small)),
-                    decodeRows(nf.packets)
-            );
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "x", 1L),
+                    decodedRow("t", "x", 2L, "", 2L)
+            ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testOversizedSingleRowRejectedBeforeReplayUsesActualEncodedSize() throws Exception {
+    public void testNullableArrayReplayKeepsNullArrayStateWithoutReflection() throws Exception {
         assertMemoryLeak(() -> {
-            String large = repeat('x', 5000);
-            List rows = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("x", 1)
-                                    .stringColumn("s", large)
-                                    .atNow(),
-                            "x", 1L,
-                            "s", large)
-            );
-            int maxDatagramSize = fullPacketSize(rows) - 1;
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longArray("la", new long[]{1, 2})
+                        .doubleArray("da", new double[]{1.0, 2.0})
+                        .atNow();
+
+                sender.stageNullLongArrayForTest("la");
+                sender.stageNullDoubleArrayForTest("da");
+                sender.longColumn("b", 3);
+
+                Assert.assertEquals(1, nf.sendCount);
+
+                sender.atNow();
+
+                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
+                Assert.assertNotNull(tableBuffer);
+                Assert.assertEquals(1, tableBuffer.getRowCount());
+
+                assertNullableArrayNullState(tableBuffer.getExistingColumn("la", TYPE_LONG_ARRAY));
+                assertNullableArrayNullState(tableBuffer.getExistingColumn("da", TYPE_DOUBLE_ARRAY));
+
+                QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("b", TYPE_LONG);
+                Assert.assertNotNull(longColumn);
+                Assert.assertEquals(1, longColumn.getSize());
+                Assert.assertEquals(1, longColumn.getValueCount());
+                Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress()));
+            }
+        });
+    }
 
+    @Test
+    public void testNullableStringPrefixFlushKeepsNullStateWithoutReflection() throws Exception {
+        assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
-                sender.table("t");
-                assertThrowsContains("single row exceeds maximum datagram size", () ->
-                        sender.longColumn("x", 1)
-                                .stringColumn("s", large)
-                                .atNow()
-                );
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+                sender.table("t")
+                        .longColumn("x", 1)
+                        .stringColumn("s", "alpha")
+                        .atNow();
+
+                sender.longColumn("x", 2);
+                sender.stringColumn("s", null);
+                sender.longColumn("b", 3);
+
+                Assert.assertEquals(1, nf.sendCount);
+
+                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
+                Assert.assertNotNull(tableBuffer);
+                Assert.assertEquals(0, tableBuffer.getRowCount());
+
+                QwpTableBuffer.ColumnBuffer stringColumn = tableBuffer.getExistingColumn("s", TYPE_STRING);
+                assertNullableStringNullState(stringColumn);
+
+                QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("x", TYPE_LONG);
+                Assert.assertNotNull(longColumn);
+                Assert.assertEquals(1, longColumn.getSize());
+                Assert.assertEquals(1, longColumn.getValueCount());
+                Assert.assertEquals(2L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress()));
+
+                QwpTableBuffer.ColumnBuffer newColumn = tableBuffer.getExistingColumn("b", TYPE_LONG);
+                Assert.assertNotNull(newColumn);
+                Assert.assertEquals(1, newColumn.getSize());
+                Assert.assertEquals(1, newColumn.getValueCount());
+                Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(newColumn.getDataAddress()));
+
+                sender.atNow();
+                sender.flush();
             }
 
-            Assert.assertEquals(0, nf.sendCount);
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqual(Arrays.asList(
+                    decodedRow("t", "x", 1L, "s", "alpha"),
+                    decodedRow("t", "x", 2L, "s", null, "b", 3L)
+            ), decodeRows(nf.packets));
         });
     }
 
@@ -993,104 +1117,181 @@ public void testOversizedArrayRowRejectedUsesActualEncodedSize() throws Exceptio
     }
 
     @Test
-    public void testIrregularArrayRejectedDuringStagingAndSenderRemainsUsable() throws Exception {
-        assertMemoryLeak(() -> {
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
-                sender.table("arrays");
-                assertThrowsContains("irregular array shape", () ->
-                        sender.doubleArray("da", new double[][]{{1.0}, {2.0, 3.0}})
-                );
-
-                sender.table("ok");
-                sender.longColumn("x", 1).atNow();
-                sender.flush();
-            }
-
-            Assert.assertEquals(1, nf.sendCount);
-            assertRowsEqual(Arrays.asList(decodedRow("ok", "x", 1L)), decodeRows(nf.packets));
-        });
-    }
-
-    @Test
-    public void testIrregularArrayRejectedDuringStagingDoesNotLeakColumnIntoSameTable() throws Exception {
+    public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() throws Exception {
         assertMemoryLeak(() -> {
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
-                sender.table("arrays");
-                assertThrowsContains("irregular array shape", () ->
-                        sender.doubleArray("bad", new double[][]{{1.0}, {2.0, 3.0}})
-                );
-
-                sender.longColumn("x", 1).atNow();
-                sender.flush();
-            }
-
-            Assert.assertEquals(1, nf.sendCount);
-            assertRowsEqual(Arrays.asList(decodedRow("arrays", "x", 1L)), decodeRows(nf.packets));
-        });
-    }
+            String large = repeat('x', 5000);
+            List oversizedRow = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("a", 2)
+                                    .stringColumn("s", large)
+                                    .atNow(),
+                            "a", 2L,
+                            "s", large)
+            );
+            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
 
-    @Test
-    public void testSchemaChangeMidRowFlushesImmediatelyAndPreservesRows() throws Exception {
-        assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
                 sender.table("t")
                         .longColumn("a", 1)
                         .atNow();
 
                 sender.longColumn("a", 2);
-                sender.longColumn("b", 3);
+                assertThrowsContains("single row exceeds maximum datagram size", () -> {
+                    sender.stringColumn("s", large);
+                    sender.atNow();
+                });
                 Assert.assertEquals(1, nf.sendCount);
 
-                sender.atNow();
-                Assert.assertEquals(1, nf.sendCount);
+                sender.cancelRow();
+                sender.longColumn("a", 3).atNow();
                 sender.flush();
             }
 
             Assert.assertEquals(2, nf.sendCount);
             assertRowsEqual(Arrays.asList(
                     decodedRow("t", "a", 1L),
-                    decodedRow("t", "a", 2L, "b", 3L)
+                    decodedRow("t", "a", 3L)
             ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplay() throws Exception {
+    public void testOversizedSingleRowRejectedAfterReplayUsesActualEncodedSize() throws Exception {
         assertMemoryLeak(() -> {
+            String small = repeat('s', 32);
+            String large = repeat('x', 5000);
+            List largeRow = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("x", 2)
+                                    .stringColumn("s", large)
+                                    .atNow(),
+                            "x", 2L,
+                            "s", large)
+            );
+            int maxDatagramSize = fullPacketSize(largeRow) - 1;
+
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
                 sender.table("t")
-                        .longColumn("a", 1)
+                        .longColumn("x", 1)
+                        .stringColumn("s", small)
                         .atNow();
 
-                sender.longColumn("a", 2);
-                sender.longColumn("b", 3);
-                Assert.assertEquals(1, nf.sendCount);
-
-                sender.stringColumn("c", "x");
-                sender.longColumn("d", 4);
-                Assert.assertEquals(1, nf.sendCount);
-
-                sender.atNow();
-                sender.flush();
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("x", 2)
+                                .stringColumn("s", large)
+                                .atNow()
+                );
             }
 
-            Assert.assertEquals(2, nf.sendCount);
-            assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L),
-                    decodedRow("t", "a", 2L, "b", 3L, "c", "x", "d", 4L)
-            ), decodeRows(nf.packets));
-        });
-    }
-
-    @Test
-    public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplayWithoutDatagramTracking() throws Exception {
+            Assert.assertEquals(1, nf.sendCount);
+            assertPacketsWithinLimit(new RunResult(nf.packets, nf.lengths, nf.sendCount), maxDatagramSize);
+            assertRowsEqual(
+                    Arrays.asList(decodedRow("t", "x", 1L, "s", small)),
+                    decodeRows(nf.packets)
+            );
+        });
+    }
+
+    @Test
+    public void testOversizedSingleRowRejectedBeforeReplayUsesActualEncodedSize() throws Exception {
         assertMemoryLeak(() -> {
+            String large = repeat('x', 5000);
+            List rows = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("x", 1)
+                                    .stringColumn("s", large)
+                                    .atNow(),
+                            "x", 1L,
+                            "s", large)
+            );
+            int maxDatagramSize = fullPacketSize(rows) - 1;
+
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("t");
+                assertThrowsContains("single row exceeds maximum datagram size", () ->
+                        sender.longColumn("x", 1)
+                                .stringColumn("s", large)
+                                .atNow()
+                );
+            }
+
+            Assert.assertEquals(0, nf.sendCount);
+        });
+    }
+
+    @Test
+    public void testRepeatedUtf8SymbolsAcrossRowsPreserveRows() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("sym", sender -> sender.table("sym")
+                                    .longColumn("x", 1)
+                                    .symbol("sym", "東京")
+                                    .atNow(),
+                            "x", 1L,
+                            "sym", "東京"),
+                    row("sym", sender -> sender.table("sym")
+                                    .longColumn("x", 2)
+                                    .symbol("sym", "東京")
+                                    .atNow(),
+                            "x", 2L,
+                            "sym", "東京"),
+                    row("sym", sender -> sender.table("sym")
+                                    .longColumn("x", 3)
+                                    .symbol("sym", "Αθηνα")
+                                    .atNow(),
+                            "x", 3L,
+                            "sym", "Αθηνα")
+            );
+
+            RunResult result = runScenario(rows, 1024 * 1024);
+            Assert.assertEquals(1, result.sendCount);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
+    @Test
+    public void testSchemaChangeAfterOutOfOrderExistingColumnsPreservesRows() throws Exception {
+        assertMemoryLeak(() -> {
+            List rows = Arrays.asList(
+                    row("schema", sender -> sender.table("schema")
+                                    .longColumn("a", 1)
+                                    .stringColumn("b", "x")
+                                    .atNow(),
+                            "a", 1L,
+                            "b", "x"),
+                    row("schema", sender -> sender.table("schema")
+                                    .symbol("c", "new")
+                                    .stringColumn("b", "y")
+                                    .longColumn("a", 2)
+                                    .atNow(),
+                            "a", 2L,
+                            "b", "y",
+                            "c", "new"),
+                    row("schema", sender -> sender.table("schema")
+                                    .symbol("c", "next")
+                                    .longColumn("a", 3)
+                                    .stringColumn("b", "z")
+                                    .atNow(),
+                            "a", 3L,
+                            "b", "z",
+                            "c", "next")
+            );
+
+            RunResult result = runScenario(rows, 1024 * 1024);
+
+            Assert.assertEquals(2, result.sendCount);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+        });
+    }
+
+    @Test
+    public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplay() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
                         .longColumn("a", 1)
                         .atNow();
@@ -1116,615 +1317,398 @@ public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplayWithoutData
     }
 
     @Test
-    public void testSchemaChangeMidRowPreservesExistingStringAndSymbolValues() throws Exception {
+    public void testSchemaChangeMidRowAllowsMultipleNewColumnsAfterReplayWithoutDatagramTracking() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
                 sender.table("t")
                         .longColumn("a", 1)
-                        .stringColumn("s", "alpha")
-                        .symbol("sym", "one")
                         .atNow();
 
                 sender.longColumn("a", 2);
-                sender.stringColumn("s", "beta");
-                sender.symbol("sym", "two");
                 sender.longColumn("b", 3);
                 Assert.assertEquals(1, nf.sendCount);
 
+                sender.stringColumn("c", "x");
+                sender.longColumn("d", 4);
+                Assert.assertEquals(1, nf.sendCount);
+
                 sender.atNow();
                 sender.flush();
             }
 
             Assert.assertEquals(2, nf.sendCount);
             assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L, "s", "alpha", "sym", "one"),
-                    decodedRow("t", "a", 2L, "s", "beta", "sym", "two", "b", 3L)
+                    decodedRow("t", "a", 1L),
+                    decodedRow("t", "a", 2L, "b", 3L, "c", "x", "d", 4L)
             ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testSchemaChangeMidRowPreservesExistingArrayValues() throws Exception {
+    public void testSchemaChangeMidRowFlushesImmediatelyAndPreservesRows() throws Exception {
         assertMemoryLeak(() -> {
-            double[] first = {1.0, 2.0};
-            double[] second = {3.5, 4.5, 5.5};
-
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
             try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
                         .longColumn("a", 1)
-                        .doubleArray("da", first)
                         .atNow();
 
                 sender.longColumn("a", 2);
-                sender.doubleArray("da", second);
                 sender.longColumn("b", 3);
                 Assert.assertEquals(1, nf.sendCount);
 
                 sender.atNow();
+                Assert.assertEquals(1, nf.sendCount);
                 sender.flush();
             }
 
             Assert.assertEquals(2, nf.sendCount);
             assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L, "da", doubleArrayValue(shape(2), first)),
-                    decodedRow("t", "a", 2L, "da", doubleArrayValue(shape(3), second), "b", 3L)
+                    decodedRow("t", "a", 1L),
+                    decodedRow("t", "a", 2L, "b", 3L)
             ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testNullableArrayReplayKeepsNullArrayStateWithoutReflection() throws Exception {
+    public void testSchemaChangeMidRowPreservesExistingArrayValues() throws Exception {
         assertMemoryLeak(() -> {
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
-                sender.table("t")
-                        .longArray("la", new long[]{1, 2})
-                        .doubleArray("da", new double[]{1.0, 2.0})
-                        .atNow();
-
-                sender.stageNullLongArrayForTest("la");
-                sender.stageNullDoubleArrayForTest("da");
-                sender.longColumn("b", 3);
-
-                Assert.assertEquals(1, nf.sendCount);
-
-                sender.atNow();
-
-                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
-                Assert.assertNotNull(tableBuffer);
-                Assert.assertEquals(1, tableBuffer.getRowCount());
-
-                assertNullableArrayNullState(tableBuffer.getExistingColumn("la", TYPE_LONG_ARRAY));
-                assertNullableArrayNullState(tableBuffer.getExistingColumn("da", TYPE_DOUBLE_ARRAY));
-
-                QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("b", TYPE_LONG);
-                Assert.assertNotNull(longColumn);
-                Assert.assertEquals(1, longColumn.getSize());
-                Assert.assertEquals(1, longColumn.getValueCount());
-                Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress()));
-            }
-        });
-    }
+            double[] first = {1.0, 2.0};
+            double[] second = {3.5, 4.5, 5.5};
 
-    @Test
-    public void testNullableStringPrefixFlushKeepsNullStateWithoutReflection() throws Exception {
-        assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
             try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
-                        .longColumn("x", 1)
-                        .stringColumn("s", "alpha")
+                        .longColumn("a", 1)
+                        .doubleArray("da", first)
                         .atNow();
 
-                sender.longColumn("x", 2);
-                sender.stringColumn("s", null);
+                sender.longColumn("a", 2);
+                sender.doubleArray("da", second);
                 sender.longColumn("b", 3);
-
                 Assert.assertEquals(1, nf.sendCount);
 
-                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
-                Assert.assertNotNull(tableBuffer);
-                Assert.assertEquals(0, tableBuffer.getRowCount());
-
-                QwpTableBuffer.ColumnBuffer stringColumn = tableBuffer.getExistingColumn("s", TYPE_STRING);
-                assertNullableStringNullState(stringColumn);
-
-                QwpTableBuffer.ColumnBuffer longColumn = tableBuffer.getExistingColumn("x", TYPE_LONG);
-                Assert.assertNotNull(longColumn);
-                Assert.assertEquals(1, longColumn.getSize());
-                Assert.assertEquals(1, longColumn.getValueCount());
-                Assert.assertEquals(2L, Unsafe.getUnsafe().getLong(longColumn.getDataAddress()));
-
-                QwpTableBuffer.ColumnBuffer newColumn = tableBuffer.getExistingColumn("b", TYPE_LONG);
-                Assert.assertNotNull(newColumn);
-                Assert.assertEquals(1, newColumn.getSize());
-                Assert.assertEquals(1, newColumn.getValueCount());
-                Assert.assertEquals(3L, Unsafe.getUnsafe().getLong(newColumn.getDataAddress()));
-
                 sender.atNow();
                 sender.flush();
             }
 
             Assert.assertEquals(2, nf.sendCount);
             assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "x", 1L, "s", "alpha"),
-                    decodedRow("t", "x", 2L, "s", null, "b", 3L)
+                    decodedRow("t", "a", 1L, "da", doubleArrayValue(shape(2), first)),
+                    decodedRow("t", "a", 2L, "da", doubleArrayValue(shape(3), second), "b", 3L)
             ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testSymbolPrefixFlushKeepsSingleRetainedDictionaryEntry() throws Exception {
+    public void testSchemaChangeMidRowPreservesExistingStringAndSymbolValues() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
             try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
-                        .symbol("sym", "alpha")
-                        .longColumn("x", 1)
+                        .longColumn("a", 1)
+                        .stringColumn("s", "alpha")
+                        .symbol("sym", "one")
                         .atNow();
 
-                sender.symbol("sym", "beta");
-                sender.longColumn("x", 2);
+                sender.longColumn("a", 2);
+                sender.stringColumn("s", "beta");
+                sender.symbol("sym", "two");
                 sender.longColumn("b", 3);
-
                 Assert.assertEquals(1, nf.sendCount);
 
-                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
-                Assert.assertNotNull(tableBuffer);
-                Assert.assertEquals(0, tableBuffer.getRowCount());
-
-                QwpTableBuffer.ColumnBuffer symbolColumn = tableBuffer.getExistingColumn("sym", TYPE_SYMBOL);
-                Assert.assertNotNull(symbolColumn);
-                Assert.assertEquals(1, symbolColumn.getSize());
-                Assert.assertEquals(1, symbolColumn.getValueCount());
-                Assert.assertEquals(1, symbolColumn.getSymbolDictionarySize());
-                Assert.assertEquals("beta", symbolColumn.getSymbolValue(0));
-                Assert.assertTrue(symbolColumn.hasSymbol("beta"));
-                Assert.assertFalse(symbolColumn.hasSymbol("alpha"));
-                Assert.assertEquals(0, Unsafe.getUnsafe().getInt(symbolColumn.getDataAddress()));
-
                 sender.atNow();
                 sender.flush();
             }
 
             Assert.assertEquals(2, nf.sendCount);
             assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "sym", "alpha", "x", 1L),
-                    decodedRow("t", "sym", "beta", "x", 2L, "b", 3L)
+                    decodedRow("t", "a", 1L, "s", "alpha", "sym", "one"),
+                    decodedRow("t", "a", 2L, "s", "beta", "sym", "two", "b", 3L)
             ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testDuplicateColumnAfterSchemaFlushReplayIsRejected() throws Exception {
+    public void testSchemaChangeWithCommittedRowsFlushesImmediately() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
             try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
                         .longColumn("a", 1)
                         .atNow();
+                Assert.assertEquals(0, nf.sendCount);
 
-                sender.longColumn("a", 2);
-                sender.longColumn("b", 3);
+                sender.longColumn("b", 2);
                 Assert.assertEquals(1, nf.sendCount);
 
-                assertThrowsContains("column 'a' already set for current row", () -> sender.longColumn("a", 4));
+                sender.atNow();
+                Assert.assertEquals(1, nf.sendCount);
 
-                sender.cancelRow();
-                sender.longColumn("a", 5).atNow();
                 sender.flush();
+                Assert.assertEquals(2, nf.sendCount);
             }
-
-            Assert.assertEquals(2, nf.sendCount);
-            assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L),
-                    decodedRow("t", "a", 5L)
-            ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testCancelRowAfterMidRowSchemaChangeDoesNotLeakSchema() throws Exception {
+    public void testSimpleLongRowUsesScatterSendPath() throws Exception {
         assertMemoryLeak(() -> {
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
-                sender.table("t")
-                        .longColumn("a", 1)
-                        .atNow();
-
-                sender.longColumn("a", 2);
-                sender.longColumn("b", 3);
-                Assert.assertEquals(1, nf.sendCount);
-
-                sender.cancelRow();
-                sender.longColumn("a", 4).atNow();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
+                sender.table("t").longColumn("x", 42L).atNow();
                 sender.flush();
             }
 
-            Assert.assertEquals(2, nf.sendCount);
-            assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L),
-                    decodedRow("t", "a", 4L)
-            ), decodeRows(nf.packets));
+            Assert.assertEquals(1, nf.sendCount);
+            Assert.assertEquals(1, nf.scatterSendCount);
+            Assert.assertEquals(0, nf.rawSendCount);
+            Assert.assertTrue("expected multiple segments for header/schema/data", nf.segmentCounts.get(0) > 1);
+            assertRowsEqual(Arrays.asList(decodedRow("t", "x", 42L)), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testUtf8StringAndSymbolStagingSupportsCancelAndPacketSizing() throws Exception {
+    public void testSwitchTableWhileRowInProgressThrowsAndPreservesRows() throws Exception {
         assertMemoryLeak(() -> {
-            String msg1 = "Gruesse 東京";
-            String msg2 = "Privet 👋 kosme";
-            List rows = Arrays.asList(
-                    row("utf8", sender -> sender.table("utf8")
-                                    .longColumn("x", 1)
-                                    .symbol("sym", "東京")
-                                    .stringColumn("msg", msg1)
-                                    .atNow(),
-                            "x", 1L,
-                            "sym", "東京",
-                            "msg", msg1),
-                    row("utf8", sender -> sender.table("utf8")
-                                    .longColumn("x", 2)
-                                    .symbol("sym", "Αθηνα")
-                                    .stringColumn("msg", msg2)
-                                    .atNow(),
-                            "x", 2L,
-                            "sym", "Αθηνα",
-                            "msg", msg2)
-            );
-            int maxDatagramSize = fullPacketSize(rows) - 1;
-
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
-                sender.table("utf8")
-                        .longColumn("x", 0)
-                        .symbol("sym", "キャンセル")
-                        .stringColumn("msg", "should not ship 👎");
-                sender.cancelRow();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) {
+                sender.table("t1").longColumn("x", 1);
 
-                sender.longColumn("x", 1)
-                        .symbol("sym", "東京")
-                        .stringColumn("msg", msg1)
-                        .atNow();
-                sender.longColumn("x", 2)
-                        .symbol("sym", "Αθηνα")
-                        .stringColumn("msg", msg2)
-                        .atNow();
+                assertThrowsContains("Cannot switch tables while row is in progress",
+                        () -> sender.table("t2"));
+
+                sender.atNow();
+                sender.table("t2").longColumn("y", 2).atNow();
                 sender.flush();
             }
 
-            RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount);
-            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
-            assertPacketsWithinLimit(result, maxDatagramSize);
-            assertRowsEqual(expectedRows(rows), decodeRows(nf.packets));
+            Assert.assertEquals(2, nf.sendCount);
+            assertRowsEqualIgnoringOrder(
+                    Arrays.asList(decodedRow("t1", "x", 1L), decodedRow("t2", "y", 2L)),
+                    decodeRows(nf.packets)
+            );
         });
     }
 
     @Test
-    public void testRepeatedUtf8SymbolsAcrossRowsPreserveRows() throws Exception {
+    public void testSymbolBoundary127To128PreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
-            List rows = Arrays.asList(
-                    row("sym", sender -> sender.table("sym")
-                                    .longColumn("x", 1)
-                                    .symbol("sym", "東京")
-                                    .atNow(),
-                            "x", 1L,
-                            "sym", "東京"),
-                    row("sym", sender -> sender.table("sym")
-                                    .longColumn("x", 2)
-                                    .symbol("sym", "東京")
-                                    .atNow(),
-                            "x", 2L,
-                            "sym", "東京"),
-                    row("sym", sender -> sender.table("sym")
-                                    .longColumn("x", 3)
-                                    .symbol("sym", "Αθηνα")
-                                    .atNow(),
-                            "x", 3L,
-                            "sym", "Αθηνα")
-            );
+            List rows = new ArrayList<>(130);
+            for (int i = 0; i < 130; i++) {
+                final int value = i;
+                rows.add(row("t", sender -> sender.table("t")
+                                .symbol("sym", "v" + value)
+                                .longColumn("x", value)
+                                .atNow(),
+                        "sym", "v" + value,
+                        "x", (long) value));
+            }
 
-            RunResult result = runScenario(rows, 1024 * 1024);
-            Assert.assertEquals(1, result.sendCount);
+            int maxDatagramSize = fullPacketSize(rows) - 1;
+            RunResult result = runScenario(rows, maxDatagramSize);
+
+            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
+            assertPacketsWithinLimit(result, maxDatagramSize);
             assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
         });
     }
 
     @Test
-    public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() throws Exception {
+    public void testSymbolBoundary16383To16384PreservesRowsAndPacketLimit() throws Exception {
         assertMemoryLeak(() -> {
-            String large = repeat('x', 5000);
-            List oversizedRow = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("a", 2)
-                                    .stringColumn("s", large)
-                                    .atNow(),
-                            "a", 2L,
-                            "s", large)
-            );
-            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
-
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
-                sender.table("t")
-                        .longColumn("a", 1)
-                        .atNow();
-
-                sender.longColumn("a", 2);
-                assertThrowsContains("single row exceeds maximum datagram size", () -> {
-                    sender.stringColumn("s", large);
-                    sender.atNow();
-                });
-                Assert.assertEquals(1, nf.sendCount);
-
-                sender.cancelRow();
-                sender.longColumn("a", 3).atNow();
-                sender.flush();
+            List rows = new ArrayList<>(16_390);
+            for (int i = 0; i < 16_390; i++) {
+                final int value = i;
+                rows.add(row("t", sender -> sender.table("t")
+                                .symbol("sym", "v" + value)
+                                .longColumn("x", value)
+                                .atNow(),
+                        "sym", "v" + value,
+                        "x", (long) value));
             }
 
-            Assert.assertEquals(2, nf.sendCount);
-            assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L),
-                    decodedRow("t", "a", 3L)
-            ), decodeRows(nf.packets));
+            int maxDatagramSize = fullPacketSize(rows) - 1;
+            RunResult result = runScenario(rows, maxDatagramSize);
+
+            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
+            assertPacketsWithinLimit(result, maxDatagramSize);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
         });
     }
 
     @Test
-    public void testAtNowOversizeFailureRollsBackWithoutExplicitCancel() throws Exception {
+    public void testSymbolPrefixFlushKeepsSingleRetainedDictionaryEntry() throws Exception {
         assertMemoryLeak(() -> {
-            String large = repeat('x', 5000);
-            List oversizedRow = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("a", 2)
-                                    .stringColumn("s", large)
-                                    .atNow(),
-                            "a", 2L,
-                            "s", large)
-            );
-            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
-
             CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
                 sender.table("t")
-                        .longColumn("a", 1)
+                        .symbol("sym", "alpha")
+                        .longColumn("x", 1)
                         .atNow();
 
-                assertThrowsContains("single row exceeds maximum datagram size", () ->
-                        sender.longColumn("a", 2)
-                                .stringColumn("s", large)
-                                .atNow()
-                );
-                Assert.assertEquals(1, nf.sendCount);
-
-                sender.longColumn("a", 3).atNow();
-                sender.flush();
-            }
-
-            Assert.assertEquals(2, nf.sendCount);
-            assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L),
-                    decodedRow("t", "a", 3L)
-            ), decodeRows(nf.packets));
-        });
-    }
-
-    @Test
-    public void testAtMicrosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
-        assertMemoryLeak(() -> {
-            String large = repeat('x', 5000);
-            List oversizedRow = Arrays.asList(
-                    row("t", sender -> sender.table("t")
-                                    .longColumn("a", 2)
-                                    .stringColumn("s", large)
-                                    .at(2_000_000L, ChronoUnit.MICROS),
-                            "a", 2L,
-                            "s", large,
-                            "", 2_000_000L)
-            );
-            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
-
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
-                sender.table("t")
-                        .longColumn("a", 1)
-                        .at(1_000_000L, ChronoUnit.MICROS);
+                sender.symbol("sym", "beta");
+                sender.longColumn("x", 2);
+                sender.longColumn("b", 3);
 
-                assertThrowsContains("single row exceeds maximum datagram size", () ->
-                        sender.longColumn("a", 2)
-                                .stringColumn("s", large)
-                                .at(2_000_000L, ChronoUnit.MICROS)
-                );
                 Assert.assertEquals(1, nf.sendCount);
 
-                sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.MICROS);
-                sender.flush();
-            }
-
-            Assert.assertEquals(2, nf.sendCount);
-            assertRowsEqual(Arrays.asList(
-                    decodedRow("t", "a", 1L, "", 1_000_000L),
-                    decodedRow("t", "a", 3L, "", 3_000_000L)
-            ), decodeRows(nf.packets));
-        });
-    }
-
-    @Test
-    public void testAtNanosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
-        assertMemoryLeak(() -> {
-            String large = repeat('x', 5000);
-            List oversizedRow = Arrays.asList(
-                    row("tn", sender -> sender.table("tn")
-                                    .longColumn("a", 2)
-                                    .stringColumn("s", large)
-                                    .at(2_000_000L, ChronoUnit.NANOS),
-                            "a", 2L,
-                            "s", large,
-                            "", 2_000_000L)
-            );
-            int maxDatagramSize = fullPacketSize(oversizedRow) - 1;
-
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
-                sender.table("tn")
-                        .longColumn("a", 1)
-                        .at(1_000_000L, ChronoUnit.NANOS);
+                QwpTableBuffer tableBuffer = sender.currentTableBufferForTest();
+                Assert.assertNotNull(tableBuffer);
+                Assert.assertEquals(0, tableBuffer.getRowCount());
 
-                assertThrowsContains("single row exceeds maximum datagram size", () ->
-                        sender.longColumn("a", 2)
-                                .stringColumn("s", large)
-                                .at(2_000_000L, ChronoUnit.NANOS)
-                );
-                Assert.assertEquals(1, nf.sendCount);
+                QwpTableBuffer.ColumnBuffer symbolColumn = tableBuffer.getExistingColumn("sym", TYPE_SYMBOL);
+                Assert.assertNotNull(symbolColumn);
+                Assert.assertEquals(1, symbolColumn.getSize());
+                Assert.assertEquals(1, symbolColumn.getValueCount());
+                Assert.assertEquals(1, symbolColumn.getSymbolDictionarySize());
+                Assert.assertEquals("beta", symbolColumn.getSymbolValue(0));
+                Assert.assertTrue(symbolColumn.hasSymbol("beta"));
+                Assert.assertFalse(symbolColumn.hasSymbol("alpha"));
+                Assert.assertEquals(0, Unsafe.getUnsafe().getInt(symbolColumn.getDataAddress()));
 
-                sender.longColumn("a", 3).at(3_000_000L, ChronoUnit.NANOS);
+                sender.atNow();
                 sender.flush();
             }
 
             Assert.assertEquals(2, nf.sendCount);
             assertRowsEqual(Arrays.asList(
-                    decodedRow("tn", "a", 1L, "", 1_000_000L),
-                    decodedRow("tn", "a", 3L, "", 3_000_000L)
+                    decodedRow("t", "sym", "alpha", "x", 1L),
+                    decodedRow("t", "sym", "beta", "x", 2L, "b", 3L)
             ), decodeRows(nf.packets));
         });
     }
 
     @Test
-    public void testSchemaChangeWithCommittedRowsFlushesImmediately() throws Exception {
+    public void testUnboundedSenderOmittedNullableAndNonNullableColumnsPreservesRows() throws Exception {
         assertMemoryLeak(() -> {
-            CapturingNetworkFacade nf = new CapturingNetworkFacade();
-            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
-                sender.table("t")
-                        .longColumn("a", 1)
-                        .atNow();
-                Assert.assertEquals(0, nf.sendCount);
-
-                sender.longColumn("b", 2);
-                Assert.assertEquals(1, nf.sendCount);
+            List rows = Arrays.asList(
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("x", 1)
+                                    .stringColumn("s", "alpha")
+                                    .symbol("sym", "one")
+                                    .atNow(),
+                            "x", 1L,
+                            "s", "alpha",
+                            "sym", "one"),
+                    row("t", sender -> sender.table("t")
+                                    .stringColumn("s", "beta")
+                                    .atNow(),
+                            "x", Long.MIN_VALUE,
+                            "s", "beta",
+                            "sym", null),
+                    row("t", sender -> sender.table("t")
+                                    .longColumn("x", 3)
+                                    .atNow(),
+                            "x", 3L,
+                            "s", null,
+                            "sym", null)
+            );
 
-                sender.atNow();
-                Assert.assertEquals(1, nf.sendCount);
+            RunResult result = runScenario(rows, 0);
 
-                sender.flush();
-                Assert.assertEquals(2, nf.sendCount);
-            }
+            Assert.assertEquals(1, result.sendCount);
+            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
         });
     }
 
     @Test
-    public void testSchemaChangeAfterOutOfOrderExistingColumnsPreservesRows() throws Exception {
+    public void testUnboundedSenderWideSchemaWithLowIndexWritePreservesRows() throws Exception {
         assertMemoryLeak(() -> {
             List rows = Arrays.asList(
-                    row("schema", sender -> sender.table("schema")
-                                    .longColumn("a", 1)
-                                    .stringColumn("b", "x")
-                                    .atNow(),
-                            "a", 1L,
-                            "b", "x"),
-                    row("schema", sender -> sender.table("schema")
-                                    .symbol("c", "new")
-                                    .stringColumn("b", "y")
-                                    .longColumn("a", 2)
+                    row("wide", sender -> sender.table("wide")
+                                    .longColumn("c0", 0)
+                                    .longColumn("c1", 1)
+                                    .longColumn("c2", 2)
+                                    .longColumn("c3", 3)
+                                    .longColumn("c4", 4)
+                                    .longColumn("c5", 5)
+                                    .longColumn("c6", 6)
+                                    .longColumn("c7", 7)
+                                    .longColumn("c8", 8)
+                                    .longColumn("c9", 9)
                                     .atNow(),
-                            "a", 2L,
-                            "b", "y",
-                            "c", "new"),
-                    row("schema", sender -> sender.table("schema")
-                                    .symbol("c", "next")
-                                    .longColumn("a", 3)
-                                    .stringColumn("b", "z")
+                            "c0", 0L,
+                            "c1", 1L,
+                            "c2", 2L,
+                            "c3", 3L,
+                            "c4", 4L,
+                            "c5", 5L,
+                            "c6", 6L,
+                            "c7", 7L,
+                            "c8", 8L,
+                            "c9", 9L),
+                    row("wide", sender -> sender.table("wide")
+                                    .longColumn("c0", 10)
                                     .atNow(),
-                            "a", 3L,
-                            "b", "z",
-                            "c", "next")
+                            "c0", 10L,
+                            "c1", Long.MIN_VALUE,
+                            "c2", Long.MIN_VALUE,
+                            "c3", Long.MIN_VALUE,
+                            "c4", Long.MIN_VALUE,
+                            "c5", Long.MIN_VALUE,
+                            "c6", Long.MIN_VALUE,
+                            "c7", Long.MIN_VALUE,
+                            "c8", Long.MIN_VALUE,
+                            "c9", Long.MIN_VALUE)
             );
 
-            RunResult result = runScenario(rows, 1024 * 1024);
+            RunResult result = runScenario(rows, 0);
 
-            Assert.assertEquals(2, result.sendCount);
+            Assert.assertEquals(1, result.sendCount);
             assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
         });
     }
 
     @Test
-    public void testBoundedSenderSchemaFlushThenOmittedNullableColumnsPreservesRows() throws Exception {
+    public void testUtf8StringAndSymbolStagingSupportsCancelAndPacketSizing() throws Exception {
         assertMemoryLeak(() -> {
+            String msg1 = "Gruesse 東京";
+            String msg2 = "Privet 👋 kosme";
             List rows = Arrays.asList(
-                    row("schema", sender -> sender.table("schema")
+                    row("utf8", sender -> sender.table("utf8")
                                     .longColumn("x", 1)
-                                    .stringColumn("s", "alpha")
+                                    .symbol("sym", "東京")
+                                    .stringColumn("msg", msg1)
                                     .atNow(),
                             "x", 1L,
-                            "s", "alpha"),
-                    row("schema", sender -> sender.table("schema")
-                                    .symbol("sym", "new")
+                            "sym", "東京",
+                            "msg", msg1),
+                    row("utf8", sender -> sender.table("utf8")
                                     .longColumn("x", 2)
-                                    .stringColumn("s", "beta")
+                                    .symbol("sym", "Αθηνα")
+                                    .stringColumn("msg", msg2)
                                     .atNow(),
-                            "sym", "new",
                             "x", 2L,
-                            "s", "beta"),
-                    row("schema", sender -> sender.table("schema")
-                                    .longColumn("x", 3)
-                                    .atNow(),
-                            "sym", null,
-                            "x", 3L,
-                            "s", null)
+                            "sym", "Αθηνα",
+                            "msg", msg2)
             );
-
-            RunResult result = runScenario(rows, 1024 * 1024);
-
-            Assert.assertEquals(2, result.sendCount);
-            assertPacketsWithinLimit(result, 1024 * 1024);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
-        });
-    }
-
-    @Test
-    public void testSymbolBoundary16383To16384PreservesRowsAndPacketLimit() throws Exception {
-        assertMemoryLeak(() -> {
-            List rows = new ArrayList<>(16_390);
-            for (int i = 0; i < 16_390; i++) {
-                final int value = i;
-                rows.add(row("t", sender -> sender.table("t")
-                                .symbol("sym", "v" + value)
-                                .longColumn("x", value)
-                                .atNow(),
-                        "sym", "v" + value,
-                        "x", (long) value));
-            }
-
             int maxDatagramSize = fullPacketSize(rows) - 1;
-            RunResult result = runScenario(rows, maxDatagramSize);
 
-            Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
-            assertPacketsWithinLimit(result, maxDatagramSize);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
-        });
-    }
+            CapturingNetworkFacade nf = new CapturingNetworkFacade();
+            try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, maxDatagramSize)) {
+                sender.table("utf8")
+                        .longColumn("x", 0)
+                        .symbol("sym", "キャンセル")
+                        .stringColumn("msg", "should not ship 👎");
+                sender.cancelRow();
 
-    @Test
-    public void testSymbolBoundary127To128PreservesRowsAndPacketLimit() throws Exception {
-        assertMemoryLeak(() -> {
-            List rows = new ArrayList<>(130);
-            for (int i = 0; i < 130; i++) {
-                final int value = i;
-                rows.add(row("t", sender -> sender.table("t")
-                                .symbol("sym", "v" + value)
-                                .longColumn("x", value)
-                                .atNow(),
-                        "sym", "v" + value,
-                        "x", (long) value));
+                sender.longColumn("x", 1)
+                        .symbol("sym", "東京")
+                        .stringColumn("msg", msg1)
+                        .atNow();
+                sender.longColumn("x", 2)
+                        .symbol("sym", "Αθηνα")
+                        .stringColumn("msg", msg2)
+                        .atNow();
+                sender.flush();
             }
 
-            int maxDatagramSize = fullPacketSize(rows) - 1;
-            RunResult result = runScenario(rows, maxDatagramSize);
-
+            RunResult result = new RunResult(nf.packets, nf.lengths, nf.sendCount);
             Assert.assertTrue("expected at least one auto-flush", result.packets.size() > 1);
             assertPacketsWithinLimit(result, maxDatagramSize);
-            assertRowsEqual(expectedRows(rows), decodeRows(result.packets));
+            assertRowsEqual(expectedRows(rows), decodeRows(nf.packets));
         });
     }
 
@@ -1742,6 +1726,43 @@ public void testZeroColumnRowsThrow() throws Exception {
         });
     }
 
+    private static void assertEstimateAtLeastActual(List rows) throws Exception {
+        CapturingNetworkFacade nf = new CapturingNetworkFacade();
+        try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
+            for (int i = 0; i < rows.size(); i++) {
+                rows.get(i).writer.accept(sender);
+                long estimate = sender.committedDatagramEstimateForTest();
+                long actual = fullPacketSize(rows.subList(0, i + 1));
+                Assert.assertTrue(
+                        "row " + i + " estimate underflow: estimate=" + estimate + ", actual=" + actual,
+                        estimate >= actual
+                );
+            }
+        }
+    }
+
+    private static void assertNullableArrayNullState(QwpTableBuffer.ColumnBuffer column) {
+        Assert.assertNotNull(column);
+        Assert.assertEquals(1, column.getSize());
+        Assert.assertEquals(0, column.getValueCount());
+        Assert.assertTrue(column.usesNullBitmap());
+        Assert.assertTrue(column.isNull(0));
+        Assert.assertEquals(0, column.getArrayShapeOffset());
+        Assert.assertEquals(0, column.getArrayDataOffset());
+    }
+
+    private static void assertNullableStringNullState(QwpTableBuffer.ColumnBuffer column) {
+        Assert.assertNotNull(column);
+        Assert.assertEquals(1, column.getSize());
+        Assert.assertEquals(0, column.getValueCount());
+        Assert.assertTrue(column.usesNullBitmap());
+        Assert.assertTrue(column.isNull(0));
+        Assert.assertEquals(0, column.getStringDataSize());
+        long offsetsAddress = column.getStringOffsetsAddress();
+        Assert.assertTrue(offsetsAddress != 0);
+        Assert.assertEquals(0, Unsafe.getUnsafe().getInt(offsetsAddress));
+    }
+
     private static void assertPacketsWithinLimit(RunResult result, int maxDatagramSize) {
         for (int i = 0; i < result.lengths.size(); i++) {
             int len = result.lengths.get(i);
@@ -1792,56 +1813,6 @@ private static void assertThrowsContains(String expected, ThrowingRunnable runna
         }
     }
 
-    private static void assertNullableArrayNullState(QwpTableBuffer.ColumnBuffer column) {
-        Assert.assertNotNull(column);
-        Assert.assertEquals(1, column.getSize());
-        Assert.assertEquals(0, column.getValueCount());
-        Assert.assertTrue(column.isNullable());
-        Assert.assertTrue(column.isNull(0));
-        Assert.assertEquals(0, column.getArrayShapeOffset());
-        Assert.assertEquals(0, column.getArrayDataOffset());
-    }
-
-    private static void assertNullableStringNullState(QwpTableBuffer.ColumnBuffer column) {
-        Assert.assertNotNull(column);
-        Assert.assertEquals(1, column.getSize());
-        Assert.assertEquals(0, column.getValueCount());
-        Assert.assertTrue(column.isNullable());
-        Assert.assertTrue(column.isNull(0));
-        Assert.assertEquals(0, column.getStringDataSize());
-        long offsetsAddress = column.getStringOffsetsAddress();
-        Assert.assertTrue(offsetsAddress != 0);
-        Assert.assertEquals(0, Unsafe.getUnsafe().getInt(offsetsAddress));
-    }
-
-    private static BigDecimal decimal(long unscaled, int scale) {
-        return BigDecimal.valueOf(unscaled, scale);
-    }
-
-    private static DecodedRow decodedRow(String table, Object... kvs) {
-        return new DecodedRow(table, columns(kvs));
-    }
-
-    private static List decodeRows(List packets) {
-        ArrayList rows = new ArrayList<>();
-        for (byte[] packet : packets) {
-            rows.addAll(new DatagramDecoder(packet).decode());
-        }
-        return rows;
-    }
-
-    private static DoubleArrayValue doubleArrayValue(int[] shape, double... values) {
-        ArrayList dims = new ArrayList<>(shape.length);
-        for (int dim : shape) {
-            dims.add(dim);
-        }
-        ArrayList elems = new ArrayList<>(values.length);
-        for (double value : values) {
-            elems.add(value);
-        }
-        return new DoubleArrayValue(dims, elems);
-    }
-
     private static void auditEstimateAcrossSymbolDictionaryVarintBoundary() throws Exception {
         ArrayList rows = new ArrayList<>();
         for (int i = 0; i < 160; i++) {
@@ -1912,19 +1883,43 @@ private static void auditEstimateWithStableSchemaAndNullableValues() throws Exce
         assertEstimateAtLeastActual(rows);
     }
 
-    private static void assertEstimateAtLeastActual(List rows) throws Exception {
-        CapturingNetworkFacade nf = new CapturingNetworkFacade();
-        try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) {
-            for (int i = 0; i < rows.size(); i++) {
-                rows.get(i).writer.accept(sender);
-                long estimate = sender.committedDatagramEstimateForTest();
-                long actual = fullPacketSize(rows.subList(0, i + 1));
-                Assert.assertTrue(
-                        "row " + i + " estimate underflow: estimate=" + estimate + ", actual=" + actual,
-                        estimate >= actual
-                );
-            }
+    private static LinkedHashMap columns(Object... kvs) {
+        if ((kvs.length & 1) != 0) {
+            throw new IllegalArgumentException("key/value pairs expected");
+        }
+        LinkedHashMap columns = new LinkedHashMap<>();
+        for (int i = 0; i < kvs.length; i += 2) {
+            columns.put((String) kvs[i], kvs[i + 1]);
+        }
+        return columns;
+    }
+
+    private static BigDecimal decimal(long unscaled, int scale) {
+        return BigDecimal.valueOf(unscaled, scale);
+    }
+
+    private static List decodeRows(List packets) {
+        ArrayList rows = new ArrayList<>();
+        for (byte[] packet : packets) {
+            rows.addAll(new DatagramDecoder(packet).decode());
+        }
+        return rows;
+    }
+
+    private static DecodedRow decodedRow(String table, Object... kvs) {
+        return new DecodedRow(table, columns(kvs));
+    }
+
+    private static DoubleArrayValue doubleArrayValue(int[] shape, double... values) {
+        ArrayList dims = new ArrayList<>(shape.length);
+        for (int dim : shape) {
+            dims.add(dim);
         }
+        ArrayList elems = new ArrayList<>(values.length);
+        for (double value : values) {
+            elems.add(value);
+        }
+        return new DoubleArrayValue(dims, elems);
     }
 
     private static List expectedRows(List rows) {
@@ -1949,12 +1944,6 @@ private static double[] flatten(double[][] matrix) {
         return flat;
     }
 
-    private static int fullPacketSize(List rows) throws Exception {
-        RunResult result = runScenario(rows, 0);
-        Assert.assertEquals("expected a single unbounded packet", 1, result.packets.size());
-        return result.lengths.get(0);
-    }
-
     private static long[] flatten(long[][] matrix) {
         int size = 0;
         for (long[] row : matrix) {
@@ -1969,15 +1958,10 @@ private static long[] flatten(long[][] matrix) {
         return flat;
     }
 
-    private static LinkedHashMap columns(Object... kvs) {
-        if ((kvs.length & 1) != 0) {
-            throw new IllegalArgumentException("key/value pairs expected");
-        }
-        LinkedHashMap columns = new LinkedHashMap<>();
-        for (int i = 0; i < kvs.length; i += 2) {
-            columns.put((String) kvs[i], kvs[i + 1]);
-        }
-        return columns;
+    private static int fullPacketSize(List rows) throws Exception {
+        RunResult result = runScenario(rows, 0);
+        Assert.assertEquals("expected a single unbounded packet", 1, result.packets.size());
+        return result.lengths.get(0);
     }
 
     private static LongArrayValue longArrayValue(int[] shape, long... values) {
@@ -2043,8 +2027,8 @@ private static final class CapturingNetworkFacade extends NetworkFacadeImpl {
         private final List packets = new ArrayList<>();
         private final List segmentCounts = new ArrayList<>();
         private int rawSendCount;
-        private int sendCount;
         private int scatterSendCount;
+        private int sendCount;
 
         @Override
         public int close(int fd) {
@@ -2131,6 +2115,16 @@ private DatagramDecoder(byte[] packet) {
             this.reader = new PacketReader(packet);
         }
 
+        private static int countNulls(boolean[] nulls) {
+            int count = 0;
+            for (boolean value : nulls) {
+                if (value) {
+                    count++;
+                }
+            }
+            return count;
+        }
+
         private List decode() {
             Assert.assertEquals(MAGIC_MESSAGE, reader.readIntLE());
             Assert.assertEquals(VERSION_1, reader.readByte());
@@ -2147,31 +2141,19 @@ private List decode() {
             return rows;
         }
 
-        private List decodeTable() {
-            String tableName = reader.readString();
-            int rowCount = (int) reader.readVarint();
-            int columnCount = (int) reader.readVarint();
-            Assert.assertEquals(SCHEMA_MODE_FULL, reader.readByte());
-
-            QwpColumnDef[] defs = new QwpColumnDef[columnCount];
-            for (int i = 0; i < columnCount; i++) {
-                defs[i] = new QwpColumnDef(reader.readString(), reader.readByte());
-            }
-
-            ColumnValues[] columns = new ColumnValues[columnCount];
-            for (int i = 0; i < columnCount; i++) {
-                columns[i] = decodeColumn(defs[i], rowCount);
-            }
-
-            ArrayList rows = new ArrayList<>(rowCount);
-            for (int row = 0; row < rowCount; row++) {
-                LinkedHashMap values = new LinkedHashMap<>();
-                for (int col = 0; col < columnCount; col++) {
-                    values.put(defs[col].getName(), columns[col].rows[row]);
+        private void decodeBooleans(Object[] values, boolean[] nulls, int valueCount) {
+            byte[] packed = reader.readBytes((valueCount + 7) / 8);
+            int valueIndex = 0;
+            for (int row = 0; row < values.length; row++) {
+                if (nulls[row]) {
+                    values[row] = null;
+                } else {
+                    int byteIndex = valueIndex >>> 3;
+                    int bitIndex = valueIndex & 7;
+                    values[row] = (packed[byteIndex] & (1 << bitIndex)) != 0;
+                    valueIndex++;
                 }
-                rows.add(new DecodedRow(tableName, values));
             }
-            return rows;
         }
 
         private ColumnValues decodeColumn(QwpColumnDef def, int rowCount) {
@@ -2220,21 +2202,6 @@ private ColumnValues decodeColumn(QwpColumnDef def, int rowCount) {
             return new ColumnValues(values);
         }
 
-        private void decodeBooleans(Object[] values, boolean[] nulls, int valueCount) {
-            byte[] packed = reader.readBytes((valueCount + 7) / 8);
-            int valueIndex = 0;
-            for (int row = 0; row < values.length; row++) {
-                if (nulls[row]) {
-                    values[row] = null;
-                } else {
-                    int byteIndex = valueIndex >>> 3;
-                    int bitIndex = valueIndex & 7;
-                    values[row] = (packed[byteIndex] & (1 << bitIndex)) != 0;
-                    valueIndex++;
-                }
-            }
-        }
-
         private void decodeDecimals(Object[] values, boolean[] nulls, int valueCount, int width) {
             int scale = reader.readByte();
             int valueIndex = 0;
@@ -2367,14 +2334,31 @@ private void decodeSymbols(Object[] values, boolean[] nulls, int valueCount) {
             Assert.assertEquals(valueCount, valueIndex);
         }
 
-        private static int countNulls(boolean[] nulls) {
-            int count = 0;
-            for (boolean value : nulls) {
-                if (value) {
-                    count++;
+        private List decodeTable() {
+            String tableName = reader.readString();
+            int rowCount = (int) reader.readVarint();
+            int columnCount = (int) reader.readVarint();
+            Assert.assertEquals(SCHEMA_MODE_FULL, reader.readByte());
+
+            QwpColumnDef[] defs = new QwpColumnDef[columnCount];
+            for (int i = 0; i < columnCount; i++) {
+                defs[i] = new QwpColumnDef(reader.readString(), reader.readByte());
+            }
+
+            ColumnValues[] columns = new ColumnValues[columnCount];
+            for (int i = 0; i < columnCount; i++) {
+                columns[i] = decodeColumn(defs[i], rowCount);
+            }
+
+            ArrayList rows = new ArrayList<>(rowCount);
+            for (int row = 0; row < rowCount; row++) {
+                LinkedHashMap values = new LinkedHashMap<>();
+                for (int col = 0; col < columnCount; col++) {
+                    values.put(defs[col].getName(), columns[col].rows[row]);
                 }
+                rows.add(new DecodedRow(tableName, values));
             }
-            return count;
+            return rows;
         }
     }
 
@@ -2491,6 +2475,14 @@ private PacketReader(byte[] data) {
             this.data = data;
         }
 
+        private int length() {
+            return data.length;
+        }
+
+        private int position() {
+            return position;
+        }
+
         private byte readByte() {
             return data[position++];
         }
@@ -2501,17 +2493,6 @@ private byte[] readBytes(int len) {
             return bytes;
         }
 
-        private boolean[] readNullBitmap(int rowCount) {
-            byte[] bitmap = readBytes((rowCount + 7) / 8);
-            boolean[] nulls = new boolean[rowCount];
-            for (int row = 0; row < rowCount; row++) {
-                int byteIndex = row >>> 3;
-                int bitIndex = row & 7;
-                nulls[row] = (bitmap[byteIndex] & (1 << bitIndex)) != 0;
-            }
-            return nulls;
-        }
-
         private double readDoubleLE() {
             double value = Unsafe.getUnsafe().getDouble(data, Unsafe.BYTE_OFFSET + position);
             position += Double.BYTES;
@@ -2530,6 +2511,17 @@ private long readLongLE() {
             return value;
         }
 
+        private boolean[] readNullBitmap(int rowCount) {
+            byte[] bitmap = readBytes((rowCount + 7) / 8);
+            boolean[] nulls = new boolean[rowCount];
+            for (int row = 0; row < rowCount; row++) {
+                int byteIndex = row >>> 3;
+                int bitIndex = row & 7;
+                nulls[row] = (bitmap[byteIndex] & (1 << bitIndex)) != 0;
+            }
+            return nulls;
+        }
+
         private String readString() {
             int len = (int) readVarint();
             if (len == 0) {
@@ -2565,14 +2557,6 @@ private long readVarint() {
                 }
             }
         }
-
-        private int length() {
-            return data.length;
-        }
-
-        private int position() {
-            return position;
-        }
     }
 
     private static final class RunResult {

From 5e0fb30c300f6b3300d05a3d79d9cb513f191ce3 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 10 Mar 2026 15:37:39 +0100
Subject: [PATCH 179/230] Remove unused method hasInProgressRow

---
 .../client/cutlass/qwp/protocol/QwpTableBuffer.java      | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index 185d4a5..67018e7 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -222,15 +222,6 @@ public String getTableName() {
         return tableName;
     }
 
-    public boolean hasInProgressRow() {
-        for (int i = 0, n = columns.size(); i < n; i++) {
-            if (fastColumns[i].size > rowCount) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     /**
      * Advances to the next row.
      * 
      

From bc2334d5d8a58b3546aebc5f881ea6467ecdf23d Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 10 Mar 2026 15:38:05 +0100
Subject: [PATCH 180/230] Simple refactoring

Rename a method, inline a method
---
 .../cutlass/qwp/client/QwpUdpSender.java      |   4 +-
 .../cutlass/qwp/protocol/QwpSchemaHash.java   |   2 +-
 .../cutlass/qwp/protocol/QwpTableBuffer.java  | 105 +++++++-----------
 3 files changed, 45 insertions(+), 66 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
index b37e93f..610924a 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java
@@ -992,7 +992,7 @@ private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) {
             }
             QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getColumn(i);
             int missing = targetRows - col.getSize();
-            if (col.isNullable()) {
+            if (col.usesNullBitmap()) {
                 estimate += bitmapBytes(targetRows) - bitmapBytes(col.getSize());
             } else {
                 estimate += nonNullablePaddingCost(col.getType(), col.getValueCount(), missing);
@@ -1293,7 +1293,7 @@ void clear() {
 
         void of(QwpTableBuffer.ColumnBuffer column) {
             this.column = column;
-            this.nullable = column.isNullable();
+            this.nullable = column.usesNullBitmap();
             this.payloadEstimateDelta = 0;
             this.sizeBefore = column.getSize();
             this.valueCountBefore = column.getValueCount();
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
index bb806cc..5105023 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
@@ -153,7 +153,7 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList>> 3;
                     nullBufPtr = Unsafe.calloc(sizeBytes, MemoryTag.NATIVE_ILP_RSS);
@@ -568,14 +567,14 @@ public ColumnBuffer(String name, byte type, boolean nullable) {
         }
 
         public void addBoolean(boolean value) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putByte(value ? (byte) 1 : (byte) 0);
             valueCount++;
             size++;
         }
 
         public void addByte(byte value) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putByte(value);
             valueCount++;
             size++;
@@ -586,7 +585,7 @@ public void addDecimal128(Decimal128 value) {
                 addNull();
                 return;
             }
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
             } else if (decimalScale != value.getScale()) {
@@ -619,7 +618,7 @@ public void addDecimal256(Decimal256 value) {
                 addNull();
                 return;
             }
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             Decimal256 src = value;
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
@@ -646,7 +645,7 @@ public void addDecimal64(Decimal64 value) {
                 addNull();
                 return;
             }
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
                 dataBuffer.putLong(value.getValue());
@@ -672,7 +671,7 @@ public void addDecimal64(Decimal64 value) {
         }
 
         public void addDouble(double value) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putDouble(value);
             valueCount++;
             size++;
@@ -779,7 +778,7 @@ public void addDoubleArrayPayload(long ptr, long len) {
         }
 
         public void addFloat(float value) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putFloat(value);
             valueCount++;
             size++;
@@ -802,28 +801,28 @@ public void addGeoHash(long value, int precision) {
                         "GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision
                 );
             }
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putLong(value);
             valueCount++;
             size++;
         }
 
         public void addInt(int value) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putInt(value);
             valueCount++;
             size++;
         }
 
         public void addLong(long value) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putLong(value);
             valueCount++;
             size++;
         }
 
         public void addLong256(long l0, long l1, long l2, long l3) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putLong(l0);
             dataBuffer.putLong(l1);
             dataBuffer.putLong(l2);
@@ -929,8 +928,8 @@ public void addLongArray(LongArray array) {
         }
 
         public void addNull() {
-            if (nullable) {
-                ensureNullCapacity(size + 1);
+            if (useNullBitmap) {
+                ensureNullBitmapCapacity();
                 markNull(size);
             } else {
                 // For non-nullable columns, store a sentinel/default value
@@ -978,18 +977,18 @@ public void addNull() {
         }
 
         public void addShort(short value) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             dataBuffer.putShort(value);
             valueCount++;
             size++;
         }
 
         public void addString(CharSequence value) {
-            if (value == null && nullable) {
-                ensureNullCapacity(size + 1);
+            if (value == null && useNullBitmap) {
+                ensureNullBitmapCapacity();
                 markNull(size);
             } else {
-                ensureNullBitmapForNonNull();
+                ensureNullBitmapCapacity();
                 if (value != null) {
                     stringData.putUtf8(value);
                 }
@@ -999,27 +998,12 @@ public void addString(CharSequence value) {
             size++;
         }
 
-        public void addStringUtf8(long ptr, int len) {
-            if (len < 0 && nullable) {
-                ensureNullCapacity(size + 1);
-                markNull(size);
-            } else {
-                ensureNullBitmapForNonNull();
-                if (len > 0) {
-                    stringData.putBlockOfBytes(ptr, len);
-                }
-                stringOffsets.putInt((int) stringData.getAppendOffset());
-                valueCount++;
-            }
-            size++;
-        }
-
         public void addSymbol(CharSequence value) {
             if (value == null) {
                 addNull();
                 return;
             }
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             int idx = getOrAddLocalSymbol(value);
             dataBuffer.putInt(idx);
             valueCount++;
@@ -1031,7 +1015,7 @@ public void addSymbolUtf8(long ptr, int len) {
                 addNull();
                 return;
             }
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             StringSink lookupSink = symbolLookupSink;
             if (lookupSink == null) {
                 symbolLookupSink = lookupSink = new StringSink(Math.max(16, len));
@@ -1054,7 +1038,7 @@ public void addSymbolWithGlobalId(String value, int globalId) {
                 addNull();
                 return;
             }
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             int localIdx = getOrAddLocalSymbol(value);
             dataBuffer.putInt(localIdx);
 
@@ -1072,7 +1056,7 @@ public void addSymbolWithGlobalId(String value, int globalId) {
         }
 
         public void addUuid(long high, long low) {
-            ensureNullBitmapForNonNull();
+            ensureNullBitmapCapacity();
             // Store in wire order: lo first, hi second
             dataBuffer.putLong(low);
             dataBuffer.putLong(high);
@@ -1254,10 +1238,6 @@ public boolean isNull(int index) {
             return (Unsafe.getUnsafe().getLong(longAddr) & (1L << bitIndex)) != 0;
         }
 
-        public boolean isNullable() {
-            return nullable;
-        }
-
         public void reset() {
             size = 0;
             valueCount = 0;
@@ -1324,7 +1304,7 @@ public void truncateTo(int newSize) {
             }
 
             int newValueCount = 0;
-            if (nullable && nullBufPtr != 0) {
+            if (useNullBitmap && nullBufPtr != 0) {
                 for (int i = 0; i < newSize; i++) {
                     if (!isNull(i)) {
                         newValueCount++;
@@ -1396,6 +1376,10 @@ public void truncateTo(int newSize) {
             }
         }
 
+        public boolean usesNullBitmap() {
+            return useNullBitmap;
+        }
+
         private static int checkedElementCount(long product) {
             if (product > Integer.MAX_VALUE) {
                 throw new LineSenderException("array too large: total element count exceeds int range");
@@ -1570,8 +1554,8 @@ private void ensureArrayCapacity(int nDims, int dataElements) {
             }
 
             // Ensure null bitmap capacity
-            if (nullable) {
-                ensureNullCapacity(size + 1);
+            if (useNullBitmap) {
+                ensureNullBitmapCapacity();
             }
 
             // Ensure shape array capacity
@@ -1599,21 +1583,16 @@ private void ensureArrayCapacity(int nDims, int dataElements) {
             }
         }
 
-        private void ensureNullBitmapForNonNull() {
-            if (nullBufPtr != 0) {
-                ensureNullCapacity(size + 1);
-            }
-        }
-
-        private void ensureNullCapacity(int rows) {
-            if (rows > nullBufCapRows) {
-                int newCapRows = Math.max(nullBufCapRows * 2, ((rows + 63) >>> 6) << 6);
-                long newSizeBytes = (long) newCapRows >>> 3;
-                long oldSizeBytes = (long) nullBufCapRows >>> 3;
-                nullBufPtr = Unsafe.realloc(nullBufPtr, oldSizeBytes, newSizeBytes, MemoryTag.NATIVE_ILP_RSS);
-                Vect.memset(nullBufPtr + oldSizeBytes, newSizeBytes - oldSizeBytes, 0);
-                nullBufCapRows = newCapRows;
+        private void ensureNullBitmapCapacity() {
+            if (nullBufPtr == 0 || nullBufCapRows > size) {
+                return;
             }
+            int newCapRows = Math.max(nullBufCapRows * 2, ((size + 64) >>> 6) << 6);
+            long newSizeBytes = (long) newCapRows >>> 3;
+            long oldSizeBytes = (long) nullBufCapRows >>> 3;
+            nullBufPtr = Unsafe.realloc(nullBufPtr, oldSizeBytes, newSizeBytes, MemoryTag.NATIVE_ILP_RSS);
+            Vect.memset(nullBufPtr + oldSizeBytes, newSizeBytes - oldSizeBytes, 0);
+            nullBufCapRows = newCapRows;
         }
 
         private int getOrAddLocalSymbol(CharSequence value) {

From 749a53f7cdf21033a14562ee1ec27f865674abf7 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 10 Mar 2026 17:25:25 +0100
Subject: [PATCH 181/230] Remove QwpTableBuffer designation as public API

---
 .../qwp/client/QwpWebSocketSender.java        | 28 ++-----------------
 1 file changed, 2 insertions(+), 26 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 7bc0e3e..b127e94 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -84,25 +84,6 @@
  *     sender.flush();
  * }
  *
      - *

      - *

      Fast-path API for high-throughput generators

      - *

      - * For maximum throughput, bypass the fluent API to avoid per-row overhead - * (no column-name hashmap lookups, no {@code checkNotClosed()}/{@code checkTableSelected()} - * per column, direct access to column buffers). The entry point is - * {@link #getTableBuffer(String)}. - * 

      - * // Setup (once)
- * QwpTableBuffer tableBuffer = sender.getTableBuffer("q");
- * QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true);
- * QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false);
- *
- * // Hot path (per row)
- * colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol));
- * colBid.addDouble(bid);
- * tableBuffer.nextRow();
- * sender.incrementPendingRowCount();
- *
      */ public class QwpWebSocketSender implements Sender { @@ -723,16 +704,11 @@ public int getPendingRowCount() { return pendingRowCount; } - /** - * DANGER:: gets or creates a low-level table buffer, - * allowing you to bypass some validation and enforcement of invariants - * present in the higher-level API. When used incorrectly, it may result - * in silent data loss. - */ + @TestOnly public QwpTableBuffer getTableBuffer(String tableName) { QwpTableBuffer buffer = tableBuffers.get(tableName); if (buffer == null) { - buffer = new QwpTableBuffer(tableName); + buffer = new QwpTableBuffer(tableName, this); tableBuffers.put(tableName, buffer); } currentTableBuffer = buffer; From 44e31b737f0d3cb84e079f343dab14b170a29c54 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 10 Mar 2026 17:26:23 +0100 Subject: [PATCH 182/230] Let QwpTableBuffer handle addSymbol fully This required injecting the Sender into QwpTableBuffer, so it can add a symbol to the global symbol table --- .../qwp/client/QwpWebSocketSender.java | 32 +++++++++++-------- .../cutlass/qwp/protocol/QwpTableBuffer.java | 29 ++++++++++++++++- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index b127e94..ef900a2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -42,6 +42,7 @@ import io.questdb.client.std.ObjList; import io.questdb.client.std.bytes.DirectByteSlice; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -696,6 +697,21 @@ public int getMaxSentSymbolId() { return maxSentSymbolId; } + /** + * Registers a symbol value in the global dictionary and returns its global ID. + * Called from {@link QwpTableBuffer.ColumnBuffer#addSymbol(CharSequence)}. + * + * @param symbol the symbol value to register + * @return the global symbol ID + */ + public int getOrAddGlobalSymbol(String symbol) { + int globalId = globalSymbolDictionary.getOrAddSymbol(symbol); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + return globalId; + } + /** * Returns the number of pending rows not yet flushed. * For testing. @@ -861,19 +877,7 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_SYMBOL, true); - - if (value != null) { - // Register symbol in global dictionary and track max ID for delta calculation - String symbolValue = value.toString(); - int globalId = globalSymbolDictionary.getOrAddSymbol(symbolValue); - if (globalId > currentBatchMaxSymbolId) { - currentBatchMaxSymbolId = globalId; - } - // Store global ID in the column buffer - col.addSymbolWithGlobalId(symbolValue, globalId); - } else { - col.addSymbol(null); - } + col.addSymbol(value); return this; } @@ -891,7 +895,7 @@ public QwpWebSocketSender table(CharSequence tableName) { currentTableName = tableName.toString(); currentTableBuffer = tableBuffers.get(currentTableName); if (currentTableBuffer == null) { - currentTableBuffer = new QwpTableBuffer(currentTableName); + currentTableBuffer = new QwpTableBuffer(currentTableName, this); tableBuffers.put(currentTableName, currentTableBuffer); } // Both modes accumulate rows until flush diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 36ecb46..be02af5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.Chars; import io.questdb.client.std.Decimal128; @@ -59,6 +60,7 @@ public class QwpTableBuffer implements QuietCloseable { private final CharSequenceIntHashMap columnNameToIndex; private final ObjList columns; + private final QwpWebSocketSender sender; private final String tableName; private QwpColumnDef[] cachedColumnDefs; private int columnAccessCursor; // tracks expected next column index @@ -70,7 +72,18 @@ public class QwpTableBuffer implements QuietCloseable { private boolean schemaHashComputed; public QwpTableBuffer(String tableName) { + this(tableName, null); + } + + /** + * Use this constructor overload to allow writing to a symbol column. + * {@link ColumnBuffer#addSymbol(CharSequence)} needs the sender to + * call {@link QwpWebSocketSender#getOrAddGlobalSymbol(String)}, registering + * the symbol in the global dictionary shared with the server. + */ + public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { this.tableName = tableName; + this.sender = sender; this.columns = new ObjList<>(); this.columnNameToIndex = new CharSequenceIntHashMap(); this.rowCount = 0; @@ -322,6 +335,7 @@ private static void assertColumnType(CharSequence name, byte type, ColumnBuffer private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) { ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); + col.sender = sender; int index = columns.size(); col.index = index; columns.add(col); @@ -534,6 +548,7 @@ public static class ColumnBuffer implements QuietCloseable { private int nullBufCapRows; // Off-heap null bitmap (bit-packed, 1 bit per row) private long nullBufPtr; + private QwpWebSocketSender sender; private int size; // Total row count (including nulls) private OffHeapAppendMemory stringData; // Off-heap storage for string/varchar column data @@ -1003,6 +1018,12 @@ public void addSymbol(CharSequence value) { addNull(); return; } + if (sender != null) { + String symbolValue = value.toString(); + int globalId = sender.getOrAddGlobalSymbol(symbolValue); + addSymbolWithGlobalId(symbolValue, globalId); + return; + } ensureNullBitmapCapacity(); int idx = getOrAddLocalSymbol(value); dataBuffer.putInt(idx); @@ -1015,7 +1036,6 @@ public void addSymbolUtf8(long ptr, int len) { addNull(); return; } - ensureNullBitmapCapacity(); StringSink lookupSink = symbolLookupSink; if (lookupSink == null) { symbolLookupSink = lookupSink = new StringSink(Math.max(16, len)); @@ -1027,6 +1047,13 @@ public void addSymbolUtf8(long ptr, int len) { Utf8s.stringFromUtf8Bytes(ptr, ptr + len); throw new AssertionError("unreachable"); } + if (sender != null) { + String symbolValue = lookupSink.toString(); + int globalId = sender.getOrAddGlobalSymbol(symbolValue); + addSymbolWithGlobalId(symbolValue, globalId); + return; + } + ensureNullBitmapCapacity(); int idx = getOrAddLocalSymbol(lookupSink); dataBuffer.putInt(idx); valueCount++; From dd6b212e7de9e5fcdee818596ed14bc8bb8b43d4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 11 Mar 2026 14:43:18 +0100 Subject: [PATCH 183/230] Move QwpSenderTest to server-side, make it self-contained --- .../cutlass/qwp/client/QwpSenderTest.java | 8346 ----------------- 1 file changed, 8346 deletions(-) delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java deleted file mode 100644 index 1629ce3..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ /dev/null @@ -1,8346 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; -import io.questdb.client.std.Decimal128; -import io.questdb.client.std.Decimal256; -import io.questdb.client.std.Decimal64; -import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.time.temporal.ChronoUnit; -import java.util.UUID; - -/** - * Integration tests for the QWP (QuestDB Wire Protocol) WebSocket sender. - *

      - * Tests verify that all QWP native types arrive correctly (exact type match) - * and that reasonable type coercions work (e.g., client sends INT but server - * column is LONG). - *

      - * Tests are skipped if no QuestDB instance is running - * ({@code -Dquestdb.running=true}). - */ -public class QwpSenderTest extends AbstractLineSenderTest { - - @BeforeClass - public static void setUpStatic() { - AbstractLineSenderTest.setUpStatic(); - } - - @Test - public void testBoolToString() throws Exception { - String table = "test_qwp_bool_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("s", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("s", false) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - true\t1970-01-01T00:00:01.000000000Z - false\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testBoolToVarchar() throws Exception { - String table = "test_qwp_bool_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("v", false) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - true\t1970-01-01T00:00:01.000000000Z - false\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testBoolean() throws Exception { - String table = "test_qwp_boolean"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("b", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("b", false) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - b\ttimestamp - true\t1970-01-01T00:00:01.000000000Z - false\t1970-01-01T00:00:02.000000000Z - """, - "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testBooleanToByteCoercionError() throws Exception { - String table = "test_qwp_boolean_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("BYTE") - ); - } - } - - @Test - public void testBooleanToCharCoercionError() throws Exception { - String table = "test_qwp_boolean_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("CHAR") - ); - } - } - - @Test - public void testBooleanToDateCoercionError() throws Exception { - String table = "test_qwp_boolean_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DATE") - ); - } - } - - @Test - public void testBooleanToDecimalCoercionError() throws Exception { - String table = "test_qwp_boolean_to_decimal_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DECIMAL") - ); - } - } - - @Test - public void testBooleanToDoubleCoercionError() throws Exception { - String table = "test_qwp_boolean_to_double_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DOUBLE") - ); - } - } - - @Test - public void testBooleanToFloatCoercionError() throws Exception { - String table = "test_qwp_boolean_to_float_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("FLOAT") - ); - } - } - - @Test - public void testBooleanToGeoHashCoercionError() throws Exception { - String table = "test_qwp_boolean_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("GEOHASH") - ); - } - } - - @Test - public void testBooleanToIntCoercionError() throws Exception { - String table = "test_qwp_boolean_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("INT") - ); - } - } - - @Test - public void testBooleanToLong256CoercionError() throws Exception { - String table = "test_qwp_boolean_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("LONG256") - ); - } - } - - @Test - public void testBooleanToLongCoercionError() throws Exception { - String table = "test_qwp_boolean_to_long_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("LONG") - ); - } - } - - @Test - public void testBooleanToShortCoercionError() throws Exception { - String table = "test_qwp_boolean_to_short_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("SHORT") - ); - } - } - - @Test - public void testBooleanToSymbolCoercionError() throws Exception { - String table = "test_qwp_boolean_to_symbol_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("SYMBOL") - ); - } - } - - @Test - public void testBooleanToTimestampCoercionError() throws Exception { - String table = "test_qwp_boolean_to_timestamp_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") - ); - } - } - - @Test - public void testBooleanToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_boolean_to_timestamp_ns_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") - ); - } - } - - @Test - public void testBooleanToUuidCoercionError() throws Exception { - String table = "test_qwp_boolean_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .boolColumn("v", true) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("UUID") - ); - } - } - - @Test - public void testByte() throws Exception { - String table = "test_qwp_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("b", (short) -1) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - } - - @Test - public void testByteToBooleanCoercionError() throws Exception { - String table = "test_qwp_byte_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("b", (byte) 1) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning BYTE and BOOLEAN but got: " + msg, - msg.contains("BYTE") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testByteToCharCoercionError() throws Exception { - String table = "test_qwp_byte_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("c", (byte) 65) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning BYTE and CHAR but got: " + msg, - msg.contains("BYTE") && msg.contains("CHAR") - ); - } - } - - @Test - public void testByteToDate() throws Exception { - String table = "test_qwp_byte_to_date"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DATE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 100) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToDecimal() throws Exception { - String table = "test_qwp_byte_to_decimal"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToDecimal128() throws Exception { - String table = "test_qwp_byte_to_decimal128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -1.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToDecimal16() throws Exception { - String table = "test_qwp_byte_to_decimal16"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -9) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -9.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToDecimal256() throws Exception { - String table = "test_qwp_byte_to_decimal256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -1.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToDecimal64() throws Exception { - String table = "test_qwp_byte_to_decimal64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -1.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToDecimal8() throws Exception { - String table = "test_qwp_byte_to_decimal8"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 5) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -9) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 5.0\t1970-01-01T00:00:01.000000000Z - -9.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToDouble() throws Exception { - String table = "test_qwp_byte_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("d", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToFloat() throws Exception { - String table = "test_qwp_byte_to_float"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("f", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("f", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - f\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT f, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToGeoHashCoercionError() throws Exception { - String table = "test_qwp_byte_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("g", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error mentioning BYTE but got: " + msg, - msg.contains("type coercion from BYTE to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testByteToInt() throws Exception { - String table = "test_qwp_byte_to_int"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("i", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("i", Byte.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("i", Byte.MIN_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - i\tts - 42\t1970-01-01T00:00:01.000000000Z - 127\t1970-01-01T00:00:02.000000000Z - -128\t1970-01-01T00:00:03.000000000Z - """, - "SELECT i, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToLong() throws Exception { - String table = "test_qwp_byte_to_long"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("l", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("l", Byte.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("l", Byte.MIN_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - l\tts - 42\t1970-01-01T00:00:01.000000000Z - 127\t1970-01-01T00:00:02.000000000Z - -128\t1970-01-01T00:00:03.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToLong256CoercionError() throws Exception { - String table = "test_qwp_byte_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("v", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from BYTE to LONG256 is not supported") - ); - } - } - - @Test - public void testByteToShort() throws Exception { - String table = "test_qwp_byte_to_short"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("s", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", Byte.MIN_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", Byte.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -128\t1970-01-01T00:00:02.000000000Z - 127\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToString() throws Exception { - String table = "test_qwp_byte_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("s", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - 0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToSymbol() throws Exception { - String table = "test_qwp_byte_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("s", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -1\t1970-01-01T00:00:02.000000000Z - 0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToTimestamp() throws Exception { - String table = "test_qwp_byte_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("t", (byte) 100) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("t", (byte) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - t\tts - 1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT t, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToUuidCoercionError() throws Exception { - String table = "test_qwp_byte_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("u", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from BYTE to UUID is not supported") - ); - } - } - - @Test - public void testByteToVarchar() throws Exception { - String table = "test_qwp_byte_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("v", (byte) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("v", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("v", Byte.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - v\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - 127\t1970-01-01T00:00:03.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testChar() throws Exception { - String table = "test_qwp_char"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("c", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("c", 'ü') // ü - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("c", '中') // 中 - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - c\ttimestamp - A\t1970-01-01T00:00:01.000000000Z - ü\t1970-01-01T00:00:02.000000000Z - 中\t1970-01-01T00:00:03.000000000Z - """, - "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testCharToBooleanCoercionError() throws Exception { - String table = "test_qwp_char_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testCharToByteCoercionError() throws Exception { - String table = "test_qwp_char_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("BYTE") - ); - } - } - - @Test - public void testCharToDateCoercionError() throws Exception { - String table = "test_qwp_char_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("DATE") - ); - } - } - - @Test - public void testCharToDoubleCoercionError() throws Exception { - String table = "test_qwp_char_to_double_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("DOUBLE") - ); - } - } - - @Test - public void testCharToFloatCoercionError() throws Exception { - String table = "test_qwp_char_to_float_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("FLOAT") - ); - } - } - - @Test - public void testCharToGeoHashCoercionError() throws Exception { - String table = "test_qwp_char_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("GEOHASH") - ); - } - } - - @Test - public void testCharToIntCoercionError() throws Exception { - String table = "test_qwp_char_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("INT") - ); - } - } - - @Test - public void testCharToLong256CoercionError() throws Exception { - String table = "test_qwp_char_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("LONG256") - ); - } - } - - @Test - public void testCharToLongCoercionError() throws Exception { - String table = "test_qwp_char_to_long_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("LONG") - ); - } - } - - @Test - public void testCharToShortCoercionError() throws Exception { - String table = "test_qwp_char_to_short_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("SHORT") - ); - } - } - - @Test - public void testCharToString() throws Exception { - String table = "test_qwp_char_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("s", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("s", 'Z') - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - A\t1970-01-01T00:00:01.000000000Z - Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testCharToSymbolCoercionError() throws Exception { - String table = "test_qwp_char_to_symbol_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write") && msg.contains("SYMBOL") - ); - } - } - - @Test - public void testCharToUuidCoercionError() throws Exception { - String table = "test_qwp_char_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("UUID") - ); - } - } - - @Test - public void testCharToVarchar() throws Exception { - String table = "test_qwp_char_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .charColumn("v", 'A') - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("v", 'Z') - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - A\t1970-01-01T00:00:01.000000000Z - Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimal() throws Exception { - String table = "test_qwp_decimal"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", "-999.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", "0.01") - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(42_000, 2)) - .at(4_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 4); - } - - @Test - public void testDecimal128ToDecimal256() throws Exception { - String table = "test_qwp_dec128_to_dec256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("d", Decimal128.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal128.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimal128ToDecimal64() throws Exception { - String table = "test_qwp_dec128_to_dec64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("d", Decimal128.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal128.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimal256ToDecimal128() throws Exception { - String table = "test_qwp_dec256_to_dec128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimal256ToDecimal64() throws Exception { - String table = "test_qwp_dec256_to_dec64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Send DECIMAL256 wire type to DECIMAL64 column - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimal256ToDecimal64OverflowError() throws Exception { - String table = "test_qwp_dec256_to_dec64_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Create a value that fits in Decimal256 but overflows Decimal64 - // Decimal256 with hi bits set will overflow 64-bit storage - Decimal256 bigValue = Decimal256.fromBigDecimal(new java.math.BigDecimal("99999999999999999999.99")); - sender.table(table) - .decimalColumn("d", bigValue) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("decimal value overflows") - ); - } - } - - @Test - public void testDecimal256ToDecimal8OverflowError() throws Exception { - String table = "test_qwp_dec256_to_dec8_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // 999.9 with scale=1 → unscaled 9999, which doesn't fit in a byte (-128..127) - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(9999, 1)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("decimal value overflows") - ); - } - } - - @Test - public void testDecimal64ToDecimal128() throws Exception { - String table = "test_qwp_dec64_to_dec128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Send DECIMAL64 wire type to DECIMAL128 column (widening) - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimal64ToDecimal256() throws Exception { - String table = "test_qwp_dec64_to_dec256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimalRescale() throws Exception { - String table = "test_qwp_decimal_rescale"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 4), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Send scale=2 wire data to scale=4 column: server should rescale - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-100, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.4500\t1970-01-01T00:00:01.000000000Z - -1.0000\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimalToBooleanCoercionError() throws Exception { - String table = "test_qwp_decimal_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testDecimalToByteCoercionError() throws Exception { - String table = "test_qwp_decimal_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("BYTE") - ); - } - } - - @Test - public void testDecimalToCharCoercionError() throws Exception { - String table = "test_qwp_decimal_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("CHAR") - ); - } - } - - @Test - public void testDecimalToDateCoercionError() throws Exception { - String table = "test_qwp_decimal_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("DATE") - ); - } - } - - @Test - public void testDecimalToDoubleCoercionError() throws Exception { - String table = "test_qwp_decimal_to_double_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("DOUBLE") - ); - } - } - - @Test - public void testDecimalToFloatCoercionError() throws Exception { - String table = "test_qwp_decimal_to_float_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("FLOAT") - ); - } - } - - @Test - public void testDecimalToGeoHashCoercionError() throws Exception { - String table = "test_qwp_decimal_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("GEOHASH") - ); - } - } - - @Test - public void testDecimalToIntCoercionError() throws Exception { - String table = "test_qwp_decimal_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("INT") - ); - } - } - - @Test - public void testDecimalToLong256CoercionError() throws Exception { - String table = "test_qwp_decimal_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("LONG256") - ); - } - } - - @Test - public void testDecimalToLongCoercionError() throws Exception { - String table = "test_qwp_decimal_to_long_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("LONG") - ); - } - } - - @Test - public void testDecimalToShortCoercionError() throws Exception { - String table = "test_qwp_decimal_to_short_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("SHORT") - ); - } - } - - @Test - public void testDecimalToString() throws Exception { - String table = "test_qwp_decimal_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("s", Decimal64.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("s", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDecimalToSymbolCoercionError() throws Exception { - String table = "test_qwp_decimal_to_symbol_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("SYMBOL") - ); - } - } - - @Test - public void testDecimalToTimestampCoercionError() throws Exception { - String table = "test_qwp_decimal_to_timestamp_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") - ); - } - } - - @Test - public void testDecimalToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_decimal_to_timestamp_ns_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") - ); - } - } - - @Test - public void testDecimalToUuidCoercionError() throws Exception { - String table = "test_qwp_decimal_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("UUID") - ); - } - } - - @Test - public void testDecimalToVarchar() throws Exception { - String table = "test_qwp_decimal_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345, 2)) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDouble() throws Exception { - String table = "test_qwp_double"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("d", 42.5) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", -1.0E10) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MIN_VALUE) - .at(4_000_000, ChronoUnit.MICROS); - // NaN and Inf should be stored as null - sender.table(table) - .doubleColumn("d", Double.NaN) - .at(5_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.POSITIVE_INFINITY) - .at(6_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.NEGATIVE_INFINITY) - .at(7_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 7); - assertSqlEventually( - """ - d\ttimestamp - 42.5\t1970-01-01T00:00:01.000000000Z - -1.0E10\t1970-01-01T00:00:02.000000000Z - 1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z - 4.9E-324\t1970-01-01T00:00:04.000000000Z - null\t1970-01-01T00:00:05.000000000Z - null\t1970-01-01T00:00:06.000000000Z - null\t1970-01-01T00:00:07.000000000Z - """, - "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testDoubleArray() throws Exception { - String table = "test_qwp_double_array"; - useTable(table); - - double[] arr1d = createDoubleArray(5); - double[][] arr2d = createDoubleArray(2, 3); - double[][][] arr3d = createDoubleArray(1, 2, 3); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleArray("a1", arr1d) - .doubleArray("a2", arr2d) - .doubleArray("a3", arr3d) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - } - - @Test - public void testDoubleArrayToIntCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("INT") - ); - } - } - - @Test - public void testDoubleArrayToStringCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_string_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("STRING") - ); - } - } - - @Test - public void testDoubleArrayToSymbolCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_symbol_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("SYMBOL") - ); - } - } - - @Test - public void testDoubleArrayToTimestampCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_timestamp_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("TIMESTAMP") - ); - } - } - - @Test - public void testDoubleToBooleanCoercionError() throws Exception { - String table = "test_qwp_double_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testDoubleToByte() throws Exception { - String table = "test_qwp_double_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("b", 42.0) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("b", -100.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - b\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToByteOverflowError() throws Exception { - String table = "test_qwp_double_to_byte_ovf"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("b", 200.0) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 200 out of range for BYTE") - ); - } - } - - @Test - public void testDoubleToBytePrecisionLossError() throws Exception { - String table = "test_qwp_double_to_byte_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("b", 42.5) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("42.5") - ); - } - } - - @Test - public void testDoubleToCharCoercionError() throws Exception { - String table = "test_qwp_double_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE") && msg.contains("CHAR") - ); - } - } - - @Test - public void testDoubleToDateCoercionError() throws Exception { - String table = "test_qwp_double_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testDoubleToDecimal() throws Exception { - String table = "test_qwp_double_to_decimal"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("d", 123.45) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", -42.10) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -42.10\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_decimal_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("d", 123.456) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("cannot be converted to") && msg.contains("123.456") && msg.contains("scale=2") - ); - } - } - - @Test - public void testDoubleToFloat() throws Exception { - String table = "test_qwp_double_to_float"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("f", 1.5) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("f", -42.25) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - } - - @Test - public void testDoubleToGeoHashCoercionError() throws Exception { - String table = "test_qwp_double_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testDoubleToInt() throws Exception { - String table = "test_qwp_double_to_int"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("i", 100_000.0) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("i", -42.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - i\tts - 100000\t1970-01-01T00:00:01.000000000Z - -42\t1970-01-01T00:00:02.000000000Z - """, - "SELECT i, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToIntPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_int_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("i", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("3.14") - ); - } - } - - @Test - public void testDoubleToLong() throws Exception { - String table = "test_qwp_double_to_long"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("l", 1_000_000.0) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("l", -42.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - l\tts - 1000000\t1970-01-01T00:00:01.000000000Z - -42\t1970-01-01T00:00:02.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToLong256CoercionError() throws Exception { - String table = "test_qwp_double_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testDoubleToShort() throws Exception { - String table = "test_qwp_double_to_short"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 100.0) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("v", -200.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - 100\t1970-01-01T00:00:01.000000000Z - -200\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToString() throws Exception { - String table = "test_qwp_double_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("s", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("s", -42.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - 3.14\t1970-01-01T00:00:01.000000000Z - -42.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToSymbol() throws Exception { - String table = "test_qwp_double_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("sym", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - sym\tts - 3.14\t1970-01-01T00:00:01.000000000Z - """, - "SELECT sym, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToUuidCoercionError() throws Exception { - String table = "test_qwp_double_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testDoubleToVarchar() throws Exception { - String table = "test_qwp_double_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("v", -42.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - 3.14\t1970-01-01T00:00:01.000000000Z - -42.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloat() throws Exception { - String table = "test_qwp_float"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("f", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("f", -42.25f) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("f", 0.0f) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - } - - @Test - public void testFloatToBooleanCoercionError() throws Exception { - String table = "test_qwp_float_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write FLOAT") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testFloatToByte() throws Exception { - String table = "test_qwp_float_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 7.0f) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("v", -100.0f) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - 7\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToCharCoercionError() throws Exception { - String table = "test_qwp_float_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write FLOAT") && msg.contains("CHAR") - ); - } - } - - @Test - public void testFloatToDateCoercionError() throws Exception { - String table = "test_qwp_float_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testFloatToDecimal() throws Exception { - String table = "test_qwp_float_to_decimal"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("d", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("d", -42.25f) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1.50\t1970-01-01T00:00:01.000000000Z - -42.25\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_decimal_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("d", 1.25f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("cannot be converted to") && msg.contains("scale=1") - ); - } - } - - @Test - public void testFloatToDouble() throws Exception { - String table = "test_qwp_float_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("d", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("d", -42.25f) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1.5\t1970-01-01T00:00:01.000000000Z - -42.25\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToGeoHashCoercionError() throws Exception { - String table = "test_qwp_float_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testFloatToInt() throws Exception { - String table = "test_qwp_float_to_int"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("i", 42.0f) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("i", -100.0f) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - i\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - """, - "SELECT i, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToIntPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_int_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("i", 3.14f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") - ); - } - } - - @Test - public void testFloatToLong() throws Exception { - String table = "test_qwp_float_to_long"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("l", 1000.0f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - l\tts - 1000\t1970-01-01T00:00:01.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToLong256CoercionError() throws Exception { - String table = "test_qwp_float_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testFloatToShort() throws Exception { - String table = "test_qwp_float_to_short"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 42.0f) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("v", -1000.0f) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - 42\t1970-01-01T00:00:01.000000000Z - -1000\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToString() throws Exception { - String table = "test_qwp_float_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("s", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - s\tts - 1.5\t1970-01-01T00:00:01.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToSymbol() throws Exception { - String table = "test_qwp_float_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("sym", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - sym\tts - 1.5\t1970-01-01T00:00:01.000000000Z - """, - "SELECT sym, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloatToUuidCoercionError() throws Exception { - String table = "test_qwp_float_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testFloatToVarchar() throws Exception { - String table = "test_qwp_float_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("v", 1.5f) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - v\tts - 1.5\t1970-01-01T00:00:01.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testInt() throws Exception { - String table = "test_qwp_int"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Integer.MIN_VALUE is the null sentinel for INT - sender.table(table) - .intColumn("i", Integer.MIN_VALUE) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", -42) - .at(4_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 4); - assertSqlEventually( - """ - i\ttimestamp - null\t1970-01-01T00:00:01.000000000Z - 0\t1970-01-01T00:00:02.000000000Z - 2147483647\t1970-01-01T00:00:03.000000000Z - -42\t1970-01-01T00:00:04.000000000Z - """, - "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testIntToBooleanCoercionError() throws Exception { - String table = "test_qwp_int_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("b", 1) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning INT and BOOLEAN but got: " + msg, - msg.contains("INT") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testIntToByte() throws Exception { - String table = "test_qwp_int_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("b", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("b", -128) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - b\tts - 42\t1970-01-01T00:00:01.000000000Z - -128\t1970-01-01T00:00:02.000000000Z - 127\t1970-01-01T00:00:03.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToByteOverflowError() throws Exception { - String table = "test_qwp_int_to_byte_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("b", 128) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") - ); - } - } - - @Test - public void testIntToCharCoercionError() throws Exception { - String table = "test_qwp_int_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("c", 65) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning INT and CHAR but got: " + msg, - msg.contains("INT") && msg.contains("CHAR") - ); - } - } - - @Test - public void testIntToDate() throws Exception { - String table = "test_qwp_int_to_date"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DATE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // 86_400_000 millis = 1 day - sender.table(table) - .intColumn("d", 86_400_000) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToDecimal() throws Exception { - String table = "test_qwp_int_to_decimal"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToDecimal128() throws Exception { - String table = "test_qwp_int_to_decimal128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - 0.00\t1970-01-01T00:00:03.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToDecimal16() throws Exception { - String table = "test_qwp_int_to_decimal16"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - 0.0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToDecimal256() throws Exception { - String table = "test_qwp_int_to_decimal256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - 0.00\t1970-01-01T00:00:03.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToDecimal64() throws Exception { - String table = "test_qwp_int_to_decimal64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("d", Integer.MAX_VALUE) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - d\tts - 2147483647.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - 0.00\t1970-01-01T00:00:03.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToDecimal8() throws Exception { - String table = "test_qwp_int_to_decimal8"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("d", 5) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -9) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - d\tts - 5.0\t1970-01-01T00:00:01.000000000Z - -9.0\t1970-01-01T00:00:02.000000000Z - 0.0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToDouble() throws Exception { - String table = "test_qwp_int_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToFloat() throws Exception { - String table = "test_qwp_int_to_float"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("f", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("f", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("f", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - f\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - 0.0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT f, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToGeoHashCoercionError() throws Exception { - String table = "test_qwp_int_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("g", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error mentioning INT but got: " + msg, - msg.contains("type coercion from INT to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testIntToLong() throws Exception { - String table = "test_qwp_int_to_long"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("l", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", Integer.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", -1) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - l\tts - 42\t1970-01-01T00:00:01.000000000Z - 2147483647\t1970-01-01T00:00:02.000000000Z - -1\t1970-01-01T00:00:03.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToLong256CoercionError() throws Exception { - String table = "test_qwp_int_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("v", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to LONG256 is not supported") - ); - } - } - - @Test - public void testIntToShort() throws Exception { - String table = "test_qwp_int_to_short"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("s", 1000) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -32768) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 32767) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 1000\t1970-01-01T00:00:01.000000000Z - -32768\t1970-01-01T00:00:02.000000000Z - 32767\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToShortOverflowError() throws Exception { - String table = "test_qwp_int_to_short_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("s", 32768) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") - ); - } - } - - @Test - public void testIntToString() throws Exception { - String table = "test_qwp_int_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("s", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - 0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToSymbol() throws Exception { - String table = "test_qwp_int_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("s", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -1\t1970-01-01T00:00:02.000000000Z - 0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToTimestamp() throws Exception { - String table = "test_qwp_int_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // 1_000_000 micros = 1 second - sender.table(table) - .intColumn("t", 1_000_000) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("t", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - t\tts - 1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT t, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToUuidCoercionError() throws Exception { - String table = "test_qwp_int_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("u", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to UUID is not supported") - ); - } - } - - @Test - public void testIntToVarchar() throws Exception { - String table = "test_qwp_int_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("v", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - v\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - 2147483647\t1970-01-01T00:00:03.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLong() throws Exception { - String table = "test_qwp_long"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Long.MIN_VALUE is the null sentinel for LONG - sender.table(table) - .longColumn("l", Long.MIN_VALUE) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", Long.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - l\ttimestamp - null\t1970-01-01T00:00:01.000000000Z - 0\t1970-01-01T00:00:02.000000000Z - 9223372036854775807\t1970-01-01T00:00:03.000000000Z - """, - "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testLong256() throws Exception { - String table = "test_qwp_long256"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // All zeros - sender.table(table) - .long256Column("v", 0, 0, 0, 0) - .at(1_000_000, ChronoUnit.MICROS); - // Mixed values - sender.table(table) - .long256Column("v", 1, 2, 3, 4) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - } - - @Test - public void testLong256ToBooleanCoercionError() throws Exception { - String table = "test_qwp_long256_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testLong256ToByteCoercionError() throws Exception { - String table = "test_qwp_long256_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToCharCoercionError() throws Exception { - String table = "test_qwp_long256_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("CHAR") - ); - } - } - - @Test - public void testLong256ToDateCoercionError() throws Exception { - String table = "test_qwp_long256_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToDoubleCoercionError() throws Exception { - String table = "test_qwp_long256_to_double_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToFloatCoercionError() throws Exception { - String table = "test_qwp_long256_to_float_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long256_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToIntCoercionError() throws Exception { - String table = "test_qwp_long256_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToLongCoercionError() throws Exception { - String table = "test_qwp_long256_to_long_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToShortCoercionError() throws Exception { - String table = "test_qwp_long256_to_short_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToString() throws Exception { - String table = "test_qwp_long256_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("s", 1, 2, 3, 4) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - s\tts - 0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z - """, - "SELECT s, ts FROM " + table); - } - - @Test - public void testLong256ToSymbolCoercionError() throws Exception { - String table = "test_qwp_long256_to_symbol_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("SYMBOL") - ); - } - } - - @Test - public void testLong256ToUuidCoercionError() throws Exception { - String table = "test_qwp_long256_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLong256ToVarchar() throws Exception { - String table = "test_qwp_long256_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .long256Column("v", 1, 2, 3, 4) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - v\tts - 0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z - """, - "SELECT v, ts FROM " + table); - } - - @Test - public void testLongToBooleanCoercionError() throws Exception { - String table = "test_qwp_long_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("b", 1) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning LONG and BOOLEAN but got: " + msg, - msg.contains("LONG") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testLongToByte() throws Exception { - String table = "test_qwp_long_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("b", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("b", -128) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - b\tts - 42\t1970-01-01T00:00:01.000000000Z - -128\t1970-01-01T00:00:02.000000000Z - 127\t1970-01-01T00:00:03.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToByteOverflowError() throws Exception { - String table = "test_qwp_long_to_byte_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("b", 128) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") - ); - } - } - - @Test - public void testLongToCharCoercionError() throws Exception { - String table = "test_qwp_long_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("c", 65) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning LONG and CHAR but got: " + msg, - msg.contains("LONG") && msg.contains("CHAR") - ); - } - } - - @Test - public void testLongToDate() throws Exception { - String table = "test_qwp_long_to_date"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DATE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", 86_400_000L) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", 0L) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToDecimal() throws Exception { - String table = "test_qwp_long_to_decimal"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToDecimal128() throws Exception { - String table = "test_qwp_long_to_decimal128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", 1_000_000_000L) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -1_000_000_000L) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1000000000.00\t1970-01-01T00:00:01.000000000Z - -1000000000.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToDecimal16() throws Exception { - String table = "test_qwp_long_to_decimal16"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToDecimal256() throws Exception { - String table = "test_qwp_long_to_decimal256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", Long.MAX_VALUE) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -1_000_000_000_000L) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 9223372036854775807.00\t1970-01-01T00:00:01.000000000Z - -1000000000000.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToDecimal32() throws Exception { - String table = "test_qwp_long_to_decimal32"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToDecimal8() throws Exception { - String table = "test_qwp_long_to_decimal8"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", 5) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -9) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 5.0\t1970-01-01T00:00:01.000000000Z - -9.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToDouble() throws Exception { - String table = "test_qwp_long_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("d", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToFloat() throws Exception { - String table = "test_qwp_long_to_float"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("f", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("f", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - f\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT f, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("g", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error mentioning LONG but got: " + msg, - msg.contains("type coercion from LONG to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testLongToInt() throws Exception { - String table = "test_qwp_long_to_int"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Value in INT range should succeed - sender.table(table) - .longColumn("i", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("i", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - i\tts - 42\t1970-01-01T00:00:01.000000000Z - -1\t1970-01-01T00:00:02.000000000Z - """, - "SELECT i, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToIntOverflowError() throws Exception { - String table = "test_qwp_long_to_int_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("i", (long) Integer.MAX_VALUE + 1) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 2147483648 out of range for INT") - ); - } - } - - @Test - public void testLongToLong256CoercionError() throws Exception { - String table = "test_qwp_long_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("v", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to LONG256 is not supported") - ); - } - } - - @Test - public void testLongToShort() throws Exception { - String table = "test_qwp_long_to_short"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Value in SHORT range should succeed - sender.table(table) - .longColumn("s", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - } - - @Test - public void testLongToShortOverflowError() throws Exception { - String table = "test_qwp_long_to_short_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("s", 32768) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") - ); - } - } - - @Test - public void testLongToString() throws Exception { - String table = "test_qwp_long_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("s", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("s", Long.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - 9223372036854775807\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToSymbol() throws Exception { - String table = "test_qwp_long_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("s", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -1\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToTimestamp() throws Exception { - String table = "test_qwp_long_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("t", 1_000_000L) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("t", 0L) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - t\tts - 1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT t, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testLongToUuidCoercionError() throws Exception { - String table = "test_qwp_long_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("u", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to UUID is not supported") - ); - } - } - - @Test - public void testLongToVarchar() throws Exception { - String table = "test_qwp_long_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .longColumn("v", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("v", Long.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - 42\t1970-01-01T00:00:01.000000000Z - 9223372036854775807\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testMultipleRowsAndBatching() throws Exception { - String table = "test_qwp_multiple_rows"; - useTable(table); - - int rowCount = 1000; - try (QwpWebSocketSender sender = createQwpSender()) { - for (int i = 0; i < rowCount; i++) { - sender.table(table) - .symbol("sym", "s" + (i % 10)) - .longColumn("val", i) - .doubleColumn("dbl", i * 1.5) - .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); - } - sender.flush(); - } - - assertTableSizeEventually(table, rowCount); - } - - @Test - public void testNullStringToBoolean() throws Exception { - String table = "test_qwp_null_string_to_boolean"; - useTable(table); - execute("CREATE TABLE " + table + " (b BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("b", "true") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - b\tts - true\t1970-01-01T00:00:01.000000000Z - false\t1970-01-01T00:00:02.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToByte() throws Exception { - String table = "test_qwp_null_string_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (b BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("b", "42") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - b\tts - 42\t1970-01-01T00:00:01.000000000Z - 0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToChar() throws Exception { - String table = "test_qwp_null_string_to_char"; - useTable(table); - execute("CREATE TABLE " + table + " (c CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("c", "A") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("c", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - c\tts - A\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT c, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToDate() throws Exception { - String table = "test_qwp_null_string_to_date"; - useTable(table); - execute("CREATE TABLE " + table + " (d DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "2022-02-25T00:00:00.000Z") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToDecimal() throws Exception { - String table = "test_qwp_null_string_to_decimal"; - useTable(table); - execute("CREATE TABLE " + table + " (d DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToFloat() throws Exception { - String table = "test_qwp_null_string_to_float"; - useTable(table); - execute("CREATE TABLE " + table + " (f FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("f", "3.14") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("f", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - f\tts - 3.14\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT f, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToGeoHash() throws Exception { - String table = "test_qwp_null_string_to_geohash"; - useTable(table); - execute("CREATE TABLE " + table + " (g GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("g", "s09wh") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("g", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - g\tts - s09wh\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT g, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToLong256() throws Exception { - String table = "test_qwp_null_string_to_long256"; - useTable(table); - execute("CREATE TABLE " + table + " (l LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("l", "0x01") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("l", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - l\tts - 0x01\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToNumeric() throws Exception { - String table = "test_qwp_null_string_to_numeric"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "l LONG, " + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("i", "42") - .stringColumn("l", "100") - .stringColumn("d", "3.14") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("i", null) - .stringColumn("l", null) - .stringColumn("d", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - i\tl\td\tts - 42\t100\t3.14\t1970-01-01T00:00:01.000000000Z - null\tnull\tnull\t1970-01-01T00:00:02.000000000Z - """, - "SELECT i, l, d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToShort() throws Exception { - String table = "test_qwp_null_string_to_short"; - useTable(table); - execute("CREATE TABLE " + table + " (s SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("s", "42") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - 0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToSymbol() throws Exception { - String table = "test_qwp_null_string_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("s", "alpha") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - alpha\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToTimestamp() throws Exception { - String table = "test_qwp_null_string_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (t TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("t", "2022-02-25T00:00:00.000000Z") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("t", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - t\tts - 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT t, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToTimestampNs() throws Exception { - String table = "test_qwp_null_string_to_timestamp_ns"; - useTable(table); - execute("CREATE TABLE " + table + " (t TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("t", "2022-02-25T00:00:00.000000Z") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("t", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - t\tts - 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT t, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToUuid() throws Exception { - String table = "test_qwp_null_string_to_uuid"; - useTable(table); - execute("CREATE TABLE " + table + " (u UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("u", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - u\tts - a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT u, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullStringToVarchar() throws Exception { - String table = "test_qwp_null_string_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("v", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - hello\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullSymbolToString() throws Exception { - String table = "test_qwp_null_symbol_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (s STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("s", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - hello\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullSymbolToSymbol() throws Exception { - String table = "test_qwp_null_symbol_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("s", "alpha") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - alpha\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testNullSymbolToVarchar() throws Exception { - String table = "test_qwp_null_symbol_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("v", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - hello\t1970-01-01T00:00:01.000000000Z - null\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testSenderBuilderWebSocket() throws Exception { - String table = "test_qwp_sender_builder_ws"; - useTable(table); - - try (Sender sender = Sender.builder(Sender.Transport.WEBSOCKET) - .address(getQuestDbHost() + ":" + getHttpPort()) - .build()) { - sender.table(table) - .symbol("city", "London") - .doubleColumn("temp", 22.5) - .longColumn("humidity", 48) - .boolColumn("sunny", true) - .stringColumn("note", "clear sky") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("city", "Berlin") - .doubleColumn("temp", 18.3) - .longColumn("humidity", 65) - .boolColumn("sunny", false) - .stringColumn("note", "overcast") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - city\ttemp\thumidity\tsunny\tnote\ttimestamp - London\t22.5\t48\ttrue\tclear sky\t1970-01-01T00:00:01.000000000Z - Berlin\t18.3\t65\tfalse\tovercast\t1970-01-01T00:00:02.000000000Z - """, - "SELECT city, temp, humidity, sunny, note, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testShort() throws Exception { - String table = "test_qwp_short"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // Short.MIN_VALUE is the null sentinel for SHORT - sender.table(table) - .shortColumn("s", Short.MIN_VALUE) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - } - - @Test - public void testShortToBooleanCoercionError() throws Exception { - String table = "test_qwp_short_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("b", (short) 1) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning SHORT and BOOLEAN but got: " + msg, - msg.contains("SHORT") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testShortToByte() throws Exception { - String table = "test_qwp_short_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("b", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) -128) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - b\tts - 42\t1970-01-01T00:00:01.000000000Z - -128\t1970-01-01T00:00:02.000000000Z - 127\t1970-01-01T00:00:03.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToByteOverflowError() throws Exception { - String table = "test_qwp_short_to_byte_overflow"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("b", (short) 128) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") - ); - } - } - - @Test - public void testShortToCharCoercionError() throws Exception { - String table = "test_qwp_short_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("c", (short) 65) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning SHORT and CHAR but got: " + msg, - msg.contains("SHORT") && msg.contains("CHAR") - ); - } - } - - @Test - public void testShortToDate() throws Exception { - String table = "test_qwp_short_to_date"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DATE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // 1000 millis = 1 second - sender.table(table) - .shortColumn("d", (short) 1000) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToDecimal128() throws Exception { - String table = "test_qwp_short_to_decimal128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("d", Short.MAX_VALUE) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", Short.MIN_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 32767.00\t1970-01-01T00:00:01.000000000Z - -32768.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToDecimal16() throws Exception { - String table = "test_qwp_short_to_decimal16"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("d", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToDecimal256() throws Exception { - String table = "test_qwp_short_to_decimal256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("d", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToDecimal32() throws Exception { - String table = "test_qwp_short_to_decimal32"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("d", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToDecimal64() throws Exception { - String table = "test_qwp_short_to_decimal64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("d", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.00\t1970-01-01T00:00:01.000000000Z - -100.00\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToDecimal8() throws Exception { - String table = "test_qwp_short_to_decimal8"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("d", (short) 5) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -9) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 5.0\t1970-01-01T00:00:01.000000000Z - -9.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToDouble() throws Exception { - String table = "test_qwp_short_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("d", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToFloat() throws Exception { - String table = "test_qwp_short_to_float"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("f", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("f", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - f\tts - 42.0\t1970-01-01T00:00:01.000000000Z - -100.0\t1970-01-01T00:00:02.000000000Z - """, - "SELECT f, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToGeoHashCoercionError() throws Exception { - String table = "test_qwp_short_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("g", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error mentioning SHORT but got: " + msg, - msg.contains("type coercion from SHORT to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testShortToInt() throws Exception { - String table = "test_qwp_short_to_int"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("i", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("i", Short.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - i\tts - 42\t1970-01-01T00:00:01.000000000Z - 32767\t1970-01-01T00:00:02.000000000Z - """, - "SELECT i, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToLong() throws Exception { - String table = "test_qwp_short_to_long"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("l", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("l", Short.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - l\tts - 42\t1970-01-01T00:00:01.000000000Z - 32767\t1970-01-01T00:00:02.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToLong256CoercionError() throws Exception { - String table = "test_qwp_short_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("v", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from SHORT to LONG256 is not supported") - ); - } - } - - @Test - public void testShortToString() throws Exception { - String table = "test_qwp_short_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("s", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - 0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToSymbol() throws Exception { - String table = "test_qwp_short_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("s", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 42\t1970-01-01T00:00:01.000000000Z - -1\t1970-01-01T00:00:02.000000000Z - 0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToTimestamp() throws Exception { - String table = "test_qwp_short_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("t", (short) 1000) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("t", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - t\tts - 1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z - 1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z - """, - "SELECT t, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testShortToUuidCoercionError() throws Exception { - String table = "test_qwp_short_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("u", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from SHORT to UUID is not supported") - ); - } - } - - @Test - public void testShortToVarchar() throws Exception { - String table = "test_qwp_short_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .shortColumn("v", (short) 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("v", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("v", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - v\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - 32767\t1970-01-01T00:00:03.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testString() throws Exception { - String table = "test_qwp_string"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("s", "hello world") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "non-ascii äöü") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "") - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", null) - .at(4_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 4); - assertSqlEventually( - """ - s\ttimestamp - hello world\t1970-01-01T00:00:01.000000000Z - non-ascii äöü\t1970-01-01T00:00:02.000000000Z - \t1970-01-01T00:00:03.000000000Z - null\t1970-01-01T00:00:04.000000000Z - """, - "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testStringToBoolean() throws Exception { - String table = "test_qwp_string_to_boolean"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("b", "true") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "false") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "1") - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "0") - .at(4_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "TRUE") - .at(5_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 5); - assertSqlEventually( - """ - b\tts - true\t1970-01-01T00:00:01.000000000Z - false\t1970-01-01T00:00:02.000000000Z - true\t1970-01-01T00:00:03.000000000Z - false\t1970-01-01T00:00:04.000000000Z - true\t1970-01-01T00:00:05.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToBooleanParseError() throws Exception { - String table = "test_qwp_string_to_boolean_err"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("b", "yes") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse boolean from string") - ); - } - } - - @Test - public void testStringToByte() throws Exception { - String table = "test_qwp_string_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("b", "42") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "-128") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "127") - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - b\tts - 42\t1970-01-01T00:00:01.000000000Z - -128\t1970-01-01T00:00:02.000000000Z - 127\t1970-01-01T00:00:03.000000000Z - """, - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToByteParseError() throws Exception { - String table = "test_qwp_string_to_byte_err"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("b", "abc") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse BYTE from string") - ); - } - } - - @Test - public void testStringToChar() throws Exception { - String table = "test_qwp_string_to_char"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("c", "A") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("c", "Hello") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - c\tts - A\t1970-01-01T00:00:01.000000000Z - H\t1970-01-01T00:00:02.000000000Z - """, - "SELECT c, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDate() throws Exception { - String table = "test_qwp_string_to_date"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DATE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "2022-02-25T00:00:00.000Z") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - d\tts - 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDateParseError() throws Exception { - String table = "test_qwp_string_to_date_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_date") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse DATE from string") && msg.contains("not_a_date") - ); - } - } - - @Test - public void testStringToDecimal128() throws Exception { - String table = "test_qwp_string_to_dec128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal16() throws Exception { - String table = "test_qwp_string_to_dec16"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "12.5") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.9") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 12.5\t1970-01-01T00:00:01.000000000Z - -99.9\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal256() throws Exception { - String table = "test_qwp_string_to_dec256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal32() throws Exception { - String table = "test_qwp_string_to_dec32"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "1234.56") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-999.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1234.56\t1970-01-01T00:00:01.000000000Z - -999.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal64() throws Exception { - String table = "test_qwp_string_to_dec64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 123.45\t1970-01-01T00:00:01.000000000Z - -99.99\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal8() throws Exception { - String table = "test_qwp_string_to_dec8"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "1.5") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-9.9") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 1.5\t1970-01-01T00:00:01.000000000Z - -9.9\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDouble() throws Exception { - String table = "test_qwp_string_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "3.14") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-2.718") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - d\tts - 3.14\t1970-01-01T00:00:01.000000000Z - -2.718\t1970-01-01T00:00:02.000000000Z - """, - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDoubleParseError() throws Exception { - String table = "test_qwp_string_to_double_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_number") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse DOUBLE from string") && msg.contains("not_a_number") - ); - } - } - - @Test - public void testStringToFloat() throws Exception { - String table = "test_qwp_string_to_float"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("f", "3.14") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("f", "-2.5") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - f\tts - 3.14\t1970-01-01T00:00:01.000000000Z - -2.5\t1970-01-01T00:00:02.000000000Z - """, - "SELECT f, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToFloatParseError() throws Exception { - String table = "test_qwp_string_to_float_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_number") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse FLOAT from string") && msg.contains("not_a_number") - ); - } - } - - @Test - public void testStringToGeoHash() throws Exception { - String table = "test_qwp_string_to_geohash"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(5c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("g", "s24se") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("g", "u33dc") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - g\tts - s24se\t1970-01-01T00:00:01.000000000Z - u33dc\t1970-01-01T00:00:02.000000000Z - """, - "SELECT g, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToGeoHashParseError() throws Exception { - String table = "test_qwp_string_to_geohash_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "!!!") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse geohash from string") && msg.contains("!!!") - ); - } - } - - @Test - public void testStringToInt() throws Exception { - String table = "test_qwp_string_to_int"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("i", "42") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("i", "-100") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("i", "0") - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - i\tts - 42\t1970-01-01T00:00:01.000000000Z - -100\t1970-01-01T00:00:02.000000000Z - 0\t1970-01-01T00:00:03.000000000Z - """, - "SELECT i, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToIntParseError() throws Exception { - String table = "test_qwp_string_to_int_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_number") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse INT from string") && msg.contains("not_a_number") - ); - } - } - - @Test - public void testStringToLong() throws Exception { - String table = "test_qwp_string_to_long"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("l", "1000000000000") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("l", "-1") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - l\tts - 1000000000000\t1970-01-01T00:00:01.000000000Z - -1\t1970-01-01T00:00:02.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToLong256() throws Exception { - String table = "test_qwp_string_to_long256"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("l", "0x01") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - l\tts - 0x01\t1970-01-01T00:00:01.000000000Z - """, - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToLong256ParseError() throws Exception { - String table = "test_qwp_string_to_long256_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_long256") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse long256 from string") && msg.contains("not_a_long256") - ); - } - } - - @Test - public void testStringToLongParseError() throws Exception { - String table = "test_qwp_string_to_long_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_number") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse LONG from string") && msg.contains("not_a_number") - ); - } - } - - @Test - public void testStringToShort() throws Exception { - String table = "test_qwp_string_to_short"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("s", "1000") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "-32768") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "32767") - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\tts - 1000\t1970-01-01T00:00:01.000000000Z - -32768\t1970-01-01T00:00:02.000000000Z - 32767\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToShortParseError() throws Exception { - String table = "test_qwp_string_to_short_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_number") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse SHORT from string") && msg.contains("not_a_number") - ); - } - } - - @Test - public void testStringToSymbol() throws Exception { - String table = "test_qwp_string_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("s", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "world") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - hello\t1970-01-01T00:00:01.000000000Z - world\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToTimestamp() throws Exception { - String table = "test_qwp_string_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("t", "2022-02-25T00:00:00.000000Z") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - t\tts - 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT t, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToTimestampNs() throws Exception { - String table = "test_qwp_string_to_timestamp_ns"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP_NS, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("ts_col", "2022-02-25T00:00:00.000000Z") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - ts_col\tts - 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT ts_col, ts FROM " + table); - } - - @Test - public void testStringToTimestampParseError() throws Exception { - String table = "test_qwp_string_to_timestamp_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not_a_timestamp") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse timestamp from string") && msg.contains("not_a_timestamp") - ); - } - } - - @Test - public void testStringToUuid() throws Exception { - String table = "test_qwp_string_to_uuid"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - u\tts - a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z - """, - "SELECT u, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToUuidParseError() throws Exception { - String table = "test_qwp_string_to_uuid_parse_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "not-a-uuid") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse UUID from string") && msg.contains("not-a-uuid") - ); - } - } - - @Test - public void testStringToVarchar() throws Exception { - String table = "test_qwp_string_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("v", "world") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - hello\t1970-01-01T00:00:01.000000000Z - world\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testSymbol() throws Exception { - String table = "test_qwp_symbol"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("s", "alpha") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", "beta") - .at(2_000_000, ChronoUnit.MICROS); - // repeated value reuses dictionary entry - sender.table(table) - .symbol("s", "alpha") - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - """ - s\ttimestamp - alpha\t1970-01-01T00:00:01.000000000Z - beta\t1970-01-01T00:00:02.000000000Z - alpha\t1970-01-01T00:00:03.000000000Z - """, - "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testSymbolToBooleanCoercionError() throws Exception { - String table = "test_qwp_symbol_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testSymbolToByteCoercionError() throws Exception { - String table = "test_qwp_symbol_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("BYTE") - ); - } - } - - @Test - public void testSymbolToCharCoercionError() throws Exception { - String table = "test_qwp_symbol_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("CHAR") - ); - } - } - - @Test - public void testSymbolToDateCoercionError() throws Exception { - String table = "test_qwp_symbol_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DATE") - ); - } - } - - @Test - public void testSymbolToDecimalCoercionError() throws Exception { - String table = "test_qwp_symbol_to_decimal_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DECIMAL") - ); - } - } - - @Test - public void testSymbolToDoubleCoercionError() throws Exception { - String table = "test_qwp_symbol_to_double_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DOUBLE") - ); - } - } - - @Test - public void testSymbolToFloatCoercionError() throws Exception { - String table = "test_qwp_symbol_to_float_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("FLOAT") - ); - } - } - - @Test - public void testSymbolToGeoHashCoercionError() throws Exception { - String table = "test_qwp_symbol_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("GEOHASH") - ); - } - } - - @Test - public void testSymbolToIntCoercionError() throws Exception { - String table = "test_qwp_symbol_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("INT") - ); - } - } - - @Test - public void testSymbolToLong256CoercionError() throws Exception { - String table = "test_qwp_symbol_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("LONG256") - ); - } - } - - @Test - public void testSymbolToLongCoercionError() throws Exception { - String table = "test_qwp_symbol_to_long_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("LONG") - ); - } - } - - @Test - public void testSymbolToShortCoercionError() throws Exception { - String table = "test_qwp_symbol_to_short_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("SHORT") - ); - } - } - - @Test - public void testSymbolToString() throws Exception { - String table = "test_qwp_symbol_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("s", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", "world") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - s\tts - hello\t1970-01-01T00:00:01.000000000Z - world\t1970-01-01T00:00:02.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testSymbolToTimestampCoercionError() throws Exception { - String table = "test_qwp_symbol_to_timestamp_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") - ); - } - } - - @Test - public void testSymbolToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_symbol_to_timestamp_ns_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") - ); - } - } - - @Test - public void testSymbolToUuidCoercionError() throws Exception { - String table = "test_qwp_symbol_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("UUID") - ); - } - } - - @Test - public void testSymbolToVarchar() throws Exception { - String table = "test_qwp_symbol_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("v", "world") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - v\tts - hello\t1970-01-01T00:00:01.000000000Z - world\t1970-01-01T00:00:02.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testTimestampMicros() throws Exception { - String table = "test_qwp_timestamp_micros"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros - sender.table(table) - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - ts_col\ttimestamp - 2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT ts_col, timestamp FROM " + table); - } - - @Test - public void testTimestampMicrosToNanos() throws Exception { - String table = "test_qwp_timestamp_micros_to_nanos"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP_NS, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_111_111L; // 2022-02-25T00:00:00Z - sender.table(table) - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - // Microseconds scaled to nanoseconds - assertSqlEventually( - """ - ts_col\tts - 2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT ts_col, ts FROM " + table); - } - - @Test - public void testTimestampNanos() throws Exception { - String table = "test_qwp_timestamp_nanos"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsNanos = 1_645_747_200_000_000_000L; // 2022-02-25T00:00:00Z in nanos - sender.table(table) - .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) - .at(tsNanos, ChronoUnit.NANOS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - } - - @Test - public void testTimestampNanosToMicros() throws Exception { - String table = "test_qwp_timestamp_nanos_to_micros"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsNanos = 1_645_747_200_123_456_789L; - sender.table(table) - .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - // Nanoseconds truncated to microseconds - assertSqlEventually( - """ - ts_col\tts - 2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT ts_col, ts FROM " + table); - } - - @Test - public void testTimestampToBooleanCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testTimestampToByteCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("BYTE") - ); - } - } - - @Test - public void testTimestampToCharCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("CHAR") - ); - } - } - - @Test - public void testTimestampToDateCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DATE") - ); - } - } - - @Test - public void testTimestampToDecimalCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_decimal_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DECIMAL") - ); - } - } - - @Test - public void testTimestampToDoubleCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_double_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DOUBLE") - ); - } - } - - @Test - public void testTimestampToFloatCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_float_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("FLOAT") - ); - } - } - - @Test - public void testTimestampToGeoHashCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("GEOHASH") - ); - } - } - - @Test - public void testTimestampToIntCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("INT") - ); - } - } - - @Test - public void testTimestampToLong256CoercionError() throws Exception { - String table = "test_qwp_timestamp_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("LONG256") - ); - } - } - - @Test - public void testTimestampToLongCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_long_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("LONG") - ); - } - } - - @Test - public void testTimestampToShortCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_short_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("SHORT") - ); - } - } - - @Test - public void testTimestampToString() throws Exception { - String table = "test_qwp_timestamp_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros - sender.table(table) - .timestampColumn("s", tsMicros, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - s\tts - 2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testTimestampToSymbolCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_symbol_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("SYMBOL") - ); - } - } - - @Test - public void testTimestampToUuidCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_uuid_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("UUID") - ); - } - } - - @Test - public void testTimestampToVarchar() throws Exception { - String table = "test_qwp_timestamp_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros - sender.table(table) - .timestampColumn("v", tsMicros, ChronoUnit.MICROS) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - v\tts - 2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z - """, - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testUuid() throws Exception { - String table = "test_qwp_uuid"; - useTable(table); - - UUID uuid1 = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .uuidColumn("u", uuid1.getLeastSignificantBits(), uuid1.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .uuidColumn("u", uuid2.getLeastSignificantBits(), uuid2.getMostSignificantBits()) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - """ - u\ttimestamp - a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z - 11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z - """, - "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testUuidToBooleanCoercionError() throws Exception { - String table = "test_qwp_uuid_to_boolean_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testUuidToByteCoercionError() throws Exception { - String table = "test_qwp_uuid_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToCharCoercionError() throws Exception { - String table = "test_qwp_uuid_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("CHAR") - ); - } - } - - @Test - public void testUuidToDateCoercionError() throws Exception { - String table = "test_qwp_uuid_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToDoubleCoercionError() throws Exception { - String table = "test_qwp_uuid_to_double_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToFloatCoercionError() throws Exception { - String table = "test_qwp_uuid_to_float_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToGeoHashCoercionError() throws Exception { - String table = "test_qwp_uuid_to_geohash_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToIntCoercionError() throws Exception { - String table = "test_qwp_uuid_to_int_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToLong256CoercionError() throws Exception { - String table = "test_qwp_uuid_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToLongCoercionError() throws Exception { - String table = "test_qwp_uuid_to_long_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); - } - } - - @Test - public void testUuidToShortCoercionError() throws Exception { - String table = "test_qwp_uuid_to_short_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to SHORT is not supported") - ); - } - } - - @Test - public void testUuidToString() throws Exception { - String table = "test_qwp_uuid_to_string"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - s\tts - a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z - """, - "SELECT s, ts FROM " + table); - } - - @Test - public void testUuidToSymbolCoercionError() throws Exception { - String table = "test_qwp_uuid_to_symbol_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("SYMBOL") - ); - } - } - - @Test - public void testUuidToVarchar() throws Exception { - String table = "test_qwp_uuid_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - """ - v\tts - a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z - """, - "SELECT v, ts FROM " + table); - } - - @Test - public void testWriteAllTypesInOneRow() throws Exception { - String table = "test_qwp_all_types"; - useTable(table); - - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - double[] arr1d = {1.0, 2.0, 3.0}; - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("sym", "test_symbol") - .boolColumn("bool_col", true) - .shortColumn("short_col", (short) 42) - .intColumn("int_col", 100_000) - .longColumn("long_col", 1_000_000_000L) - .floatColumn("float_col", 2.5f) - .doubleColumn("double_col", 3.14) - .stringColumn("string_col", "hello") - .charColumn("char_col", 'Z') - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) - .uuidColumn("uuid_col", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .long256Column("long256_col", 1, 0, 0, 0) - .doubleArray("arr_col", arr1d) - .decimalColumn("decimal_col", "99.99") - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - } - - private QwpWebSocketSender createQwpSender() { - return QwpWebSocketSender.connect(getQuestDbHost(), getHttpPort()); - } -} From 2821d91b76937fa5ab6ec533d36b19e84b02feb8 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 12 Mar 2026 12:46:31 +0100 Subject: [PATCH 184/230] Remove unmasking logic (only needed on server) --- .../cutlass/http/client/WebSocketClient.java | 5 -- .../qwp/websocket/WebSocketFrameParser.java | 64 +------------------ .../websocket/WebSocketFrameParserTest.java | 1 - 3 files changed, 2 insertions(+), 68 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 443c7a2..03b468b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -772,11 +772,6 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) { } int payloadLen = (int) payloadLength; - // Unmask if needed (server frames should not be masked) - if (frameParser.isMasked()) { - frameParser.unmaskPayload(payloadPtr, payloadLen); - } - // Handle frame by opcode int opcode = frameParser.getOpcode(); switch (opcode) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index ffd8a37..6d4a7ac 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -60,7 +60,6 @@ public class WebSocketFrameParser { // Frame header bits private static final int FIN_BIT = 0x80; private static final int LENGTH_MASK = 0x7F; - private static final int MASK_BIT = 0x80; // Control frame max payload size (RFC 6455) private static final int MAX_CONTROL_FRAME_PAYLOAD = 125; private static final int OPCODE_MASK = 0x0F; @@ -69,8 +68,6 @@ public class WebSocketFrameParser { // Parsed frame data private boolean fin; private int headerSize; - private int maskKey; - private boolean masked; private int opcode; private long payloadLength; // Parser state @@ -100,10 +97,6 @@ public boolean isFin() { return fin; } - public boolean isMasked() { - return masked; - } - /** * Parses a WebSocket frame from the given buffer. * @@ -147,15 +140,10 @@ public int parse(long buf, long limit) { return 0; } - final boolean masked = (byte1 & MASK_BIT) != 0; - this.masked = masked; int lengthField = byte1 & LENGTH_MASK; - // Validate masking based on mode - // Configuration - // If true, expect masked frames from clients - if (masked) { - // Server frames MUST NOT be masked + // Server frames MUST NOT be masked (RFC 6455 section 5.1) + if ((byte1 & 0x80) != 0) { state = STATE_ERROR; errorCode = WebSocketCloseCode.PROTOCOL_ERROR; return 0; @@ -164,7 +152,6 @@ public int parse(long buf, long limit) { // Calculate header size and payload length int offset = 2; - // If true, reject non-minimal length encodings if (lengthField <= 125) { payloadLength = lengthField; } else if (lengthField == 126) { @@ -176,9 +163,6 @@ public int parse(long buf, long limit) { int high = Unsafe.getUnsafe().getByte(buf + 2) & 0xFF; int low = Unsafe.getUnsafe().getByte(buf + 3) & 0xFF; payloadLength = (high << 8) | low; - - // Strict mode: reject non-minimal encodings - offset = 4; } else { // 64-bit extended length @@ -188,8 +172,6 @@ public int parse(long buf, long limit) { } payloadLength = Long.reverseBytes(Unsafe.getUnsafe().getLong(buf + 2)); - // Strict mode: reject non-minimal encodings - // MSB must be 0 (no negative lengths) if (payloadLength < 0) { state = STATE_ERROR; @@ -214,7 +196,6 @@ public int parse(long buf, long limit) { return 0; } - maskKey = 0; headerSize = offset; // Check if we have the complete payload @@ -235,50 +216,9 @@ public void reset() { state = STATE_HEADER; fin = false; opcode = 0; - masked = false; - maskKey = 0; payloadLength = 0; headerSize = 0; errorCode = 0; } - /** - * Unmasks the payload data in place. - * - * @param buf the start of the payload data - * @param len the length of the payload - */ - public void unmaskPayload(long buf, long len) { - if (!masked || maskKey == 0) { - // a zero maskKey is a no-op (makes no change to the data) - return; - } - - // Process 8 bytes at a time when possible for better performance - long i = 0; - long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); - - // Process 8-byte chunks - while (i + 8 <= len) { - long value = Unsafe.getUnsafe().getLong(buf + i); - Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); - i += 8; - } - - // Process 4-byte chunk if remaining - if (i + 4 <= len) { - int value = Unsafe.getUnsafe().getInt(buf + i); - Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); - i += 4; - } - - // Process remaining bytes - while (i < len) { - byte b = Unsafe.getUnsafe().getByte(buf + i); - int shift = ((int) (i % 4)) << 3; // 0, 8, 16, or 24 - byte maskByte = (byte) ((maskKey >> shift) & 0xFF); - Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); - i++; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java index dac1ec4..4a5fed9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java @@ -466,7 +466,6 @@ public void testParseMinimalFrame() throws Exception { Assert.assertTrue(parser.isFin()); Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); Assert.assertEquals(1, parser.getPayloadLength()); - Assert.assertFalse(parser.isMasked()); } finally { freeBuffer(buf, 16); } From 415fa9c3ee4ca5fac079cf1e04667e54d19edab4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 12 Mar 2026 13:30:47 +0100 Subject: [PATCH 185/230] Move tests using mock websocket server to client module --- .../QwpWebSocketAckIntegrationTest.java | 299 +++++++++++++ .../qwp/websocket/TestWebSocketServer.java | 392 ++++++++++++++++++ 2 files changed, 691 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java new file mode 100644 index 0000000..d63ff1b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java @@ -0,0 +1,299 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Integration tests for QWP v1 WebSocket ACK delivery mechanism. + * These tests verify that the InFlightWindow and ACK responses work correctly end-to-end. + */ +public class QwpWebSocketAckIntegrationTest extends AbstractTest { + + private static final int TEST_PORT = 19_500 + (int) (System.nanoTime() % 100); + + @Test + public void testAsyncFlushFailsFastOnInvalidAckPayload() throws Exception { + InvalidAckPayloadHandler handler = new InvalidAckPayloadHandler(); + int port = TEST_PORT + 21; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + boolean errorCaught = false; + long start = System.currentTimeMillis(); + try (QwpWebSocketSender sender = QwpWebSocketSender.connectAsync( + "localhost", port, false, 0, 0, 0)) { + sender.table("test") + .longColumn("value", 1) + .atNow(); + sender.flush(); + } catch (Exception e) { + errorCaught = true; + Assert.assertTrue( + e.getMessage().contains("Invalid ACK response payload") + || e.getMessage().contains("Error in send queue") + ); + } + + long duration = System.currentTimeMillis() - start; + Assert.assertTrue("Expected invalid ACK error", errorCaught); + Assert.assertTrue("Flush should fail quickly on invalid ACK [duration=" + duration + "ms]", duration < 10_000); + } + } + + @Test + public void testAsyncFlushFailsFastOnServerClose() throws Exception { + ClosingServerHandler handler = new ClosingServerHandler(); + int port = TEST_PORT + 20; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + boolean errorCaught = false; + long start = System.currentTimeMillis(); + try (QwpWebSocketSender sender = QwpWebSocketSender.connectAsync( + "localhost", port, false, 0, 0, 0)) { + sender.table("test") + .longColumn("value", 1) + .atNow(); + sender.flush(); + } catch (Exception e) { + errorCaught = true; + Assert.assertTrue( + e.getMessage().contains("closed") + || e.getMessage().contains("Error in send queue") + || e.getMessage().contains("failed") + ); + } + + long duration = System.currentTimeMillis() - start; + Assert.assertTrue("Expected async close error", errorCaught); + Assert.assertTrue("Flush should fail quickly on close [duration=" + duration + "ms]", duration < 10_000); + } + } + + /** + * Test that flush blocks until ACK is received. + * Uses async mode to enable ACK handling via InFlightWindow. + */ + @Test + public void testFlushBlocksUntilAcked() throws Exception { + final long DELAY_MS = 300; // 300ms delay before ACK + DelayedAckHandler handler = new DelayedAckHandler(DELAY_MS); + + int port = TEST_PORT + 10; + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + try (QwpWebSocketSender sender = QwpWebSocketSender.connectAsync( + "localhost", port, false, 0, 0, 0)) { + + sender.table("test") + .longColumn("value", 42) + .atNow(); + + long startTime = System.currentTimeMillis(); + sender.flush(); + long duration = System.currentTimeMillis() - startTime; + + Assert.assertTrue("Flush should have waited for ACK (took " + duration + "ms, expected >= " + (DELAY_MS / 2) + "ms)", + duration >= DELAY_MS / 2); + + LOG.info("Flush waited {}ms for ACK", duration); + } + } + } + + @Test + public void testSyncFlushFailsOnInvalidAckPayload() throws Exception { + InvalidAckPayloadHandler handler = new InvalidAckPayloadHandler(); + int port = TEST_PORT + 22; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + boolean errorCaught = false; + long start = System.currentTimeMillis(); + try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", port, false)) { + sender.table("test") + .longColumn("value", 7) + .atNow(); + sender.flush(); + } catch (Exception e) { + errorCaught = true; + Assert.assertTrue( + e.getMessage().contains("Invalid ACK response payload") + || e.getMessage().contains("Failed to parse ACK response") + ); + } + + long duration = System.currentTimeMillis() - start; + Assert.assertTrue("Expected invalid ACK error in sync mode", errorCaught); + Assert.assertTrue("Sync invalid ACK path should fail quickly [duration=" + duration + "ms]", duration < 10_000); + } + } + + @Test + public void testSyncFlushIgnoresPingAndWaitsForAck() throws Exception { + final long ackDelayMs = 300; + PingThenDelayedAckHandler handler = new PingThenDelayedAckHandler(ackDelayMs); + int port = TEST_PORT + 23; + + try (TestWebSocketServer server = new TestWebSocketServer(port, handler)) { + server.start(); + Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); + + try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", port, false)) { + sender.table("test") + .longColumn("value", 11) + .atNow(); + + long start = System.currentTimeMillis(); + sender.flush(); + long duration = System.currentTimeMillis() - start; + + Assert.assertTrue("Flush returned too early [duration=" + duration + "ms]", duration >= ackDelayMs / 2); + } + } + } + + /** + * Creates a binary ACK response using WebSocketResponse format. + * Format: status (1 byte) + sequence (8 bytes little-endian) + */ + private static byte[] createAckResponse(long sequence) { + byte[] response = new byte[WebSocketResponse.MIN_RESPONSE_SIZE]; + + // Status OK (0) + response[0] = WebSocketResponse.STATUS_OK; + + // Sequence (little-endian) + response[1] = (byte) (sequence & 0xFF); + response[2] = (byte) ((sequence >> 8) & 0xFF); + response[3] = (byte) ((sequence >> 16) & 0xFF); + response[4] = (byte) ((sequence >> 24) & 0xFF); + response[5] = (byte) ((sequence >> 32) & 0xFF); + response[6] = (byte) ((sequence >> 40) & 0xFF); + response[7] = (byte) ((sequence >> 48) & 0xFF); + response[8] = (byte) ((sequence >> 56) & 0xFF); + + return response; + } + + private static class ClosingServerHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendClose(WebSocketCloseCode.GOING_AWAY, "bye"); + } catch (IOException e) { + LOG.error("Failed to send close frame", e); + } + } + } + + /** + * Server handler that delays ACKs to test blocking behavior. + */ + private static class DelayedAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final long delayMs; + private final AtomicLong nextSequence = new AtomicLong(0); + + DelayedAckHandler(long delayMs) { + this.delayMs = delayMs; + } + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long sequence = nextSequence.getAndIncrement(); + + LOG.debug("Server delaying ACK by {}ms", delayMs); + + new Thread(() -> { + try { + Thread.sleep(delayMs); + byte[] ackResponse = createAckResponse(sequence); + client.sendBinary(ackResponse); + LOG.debug("Server sent delayed ACK for seq {}", sequence); + } catch (Exception e) { + LOG.error("Failed to send delayed ACK", e); + } + }).start(); + } + } + + private static class InvalidAckPayloadHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(new byte[]{1, 2, 3}); + } catch (IOException e) { + LOG.error("Failed to send invalid payload", e); + } + } + } + + private static class PingThenDelayedAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final long delayMs; + private final AtomicLong nextSequence = new AtomicLong(0); + + private PingThenDelayedAckHandler(long delayMs) { + this.delayMs = delayMs; + } + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long sequence = nextSequence.getAndIncrement(); + try { + client.sendPing(new byte[]{42}); + } catch (IOException e) { + LOG.error("Failed to send ping", e); + } + + new Thread(() -> { + try { + Thread.sleep(delayMs); + client.sendBinary(createAckResponse(sequence)); + } catch (Exception e) { + LOG.error("Failed to send delayed ACK", e); + } + }).start(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java new file mode 100644 index 0000000..54c5464 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -0,0 +1,392 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.websocket; + +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A simple WebSocket server for client integration testing. + * Uses plain Java heap buffers - no native memory. + */ +public class TestWebSocketServer implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(TestWebSocketServer.class); + private static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + private final List clients = new CopyOnWriteArrayList<>(); + private final WebSocketServerHandler handler; + private final int port; + private final AtomicBoolean running = new AtomicBoolean(false); + private final CountDownLatch startLatch = new CountDownLatch(1); + private Thread acceptThread; + private ServerSocket serverSocket; + + public TestWebSocketServer(int port, WebSocketServerHandler handler) { + this.port = port; + this.handler = handler; + } + + public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedException { + return startLatch.await(timeout, unit); + } + + @Override + public void close() { + running.set(false); + + for (ClientHandler client : clients) { + client.close(); + } + clients.clear(); + + if (serverSocket != null) { + try { + serverSocket.close(); + } catch (IOException e) { + // ignore + } + } + + if (acceptThread != null) { + try { + acceptThread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + public void start() throws IOException { + if (running.getAndSet(true)) { + return; + } + + serverSocket = new ServerSocket(port); + serverSocket.setSoTimeout(100); + + acceptThread = new Thread(() -> { + startLatch.countDown(); + while (running.get()) { + try { + Socket clientSocket = serverSocket.accept(); + ClientHandler clientHandler = new ClientHandler(clientSocket); + clients.add(clientHandler); + clientHandler.start(); + } catch (SocketTimeoutException e) { + // expected, check running flag + } catch (IOException e) { + if (running.get()) { + LOG.error("Accept error", e); + } + } + } + }, "WebSocket-Accept"); + acceptThread.start(); + } + + /** + * Interface for handling WebSocket server events. + */ + public interface WebSocketServerHandler { + default void onBinaryMessage(ClientHandler client, byte[] data) { + } + } + + /** + * Handles a single WebSocket client connection. + */ + public class ClientHandler implements Closeable { + private final ByteBuffer recvBuffer = ByteBuffer.allocate(65_536).order(ByteOrder.BIG_ENDIAN); + private final AtomicBoolean running = new AtomicBoolean(false); + private final Socket socket; + private InputStream in; + private boolean isClosed; + private OutputStream out; + private Thread readThread; + + ClientHandler(Socket socket) { + this.socket = socket; + recvBuffer.flip(); // start with nothing readable + } + + @Override + public void close() { + running.set(false); + try { + socket.close(); + } catch (IOException e) { + // ignore + } + if (readThread != null) { + try { + readThread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + public synchronized void sendBinary(byte[] data) throws IOException { + writeFrame(WebSocketOpcode.BINARY, data, data.length); + } + + public synchronized void sendClose(int code, String reason) throws IOException { + byte[] reasonBytes = (reason != null && !reason.isEmpty()) + ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] payload = new byte[2 + reasonBytes.length]; + payload[0] = (byte) ((code >> 8) & 0xFF); + payload[1] = (byte) (code & 0xFF); + System.arraycopy(reasonBytes, 0, payload, 2, reasonBytes.length); + writeFrame(WebSocketOpcode.CLOSE, payload, payload.length); + } + + public synchronized void sendPing(byte[] data) throws IOException { + writeFrame(WebSocketOpcode.PING, data, data.length); + } + + private String computeAcceptKey(String key) { + try { + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + sha1.update((key + WEBSOCKET_GUID).getBytes(StandardCharsets.US_ASCII)); + return Base64.getEncoder().encodeToString(sha1.digest()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void handleRead() { + while (recvBuffer.remaining() >= 2) { + recvBuffer.mark(); + + int byte0 = recvBuffer.get() & 0xFF; + int byte1 = recvBuffer.get() & 0xFF; + + int opcode = byte0 & 0x0F; + boolean isMasked = (byte1 & 0x80) != 0; + int lengthField = byte1 & 0x7F; + + long payloadLength; + if (lengthField <= 125) { + payloadLength = lengthField; + } else if (lengthField == 126) { + if (recvBuffer.remaining() < 2) { + recvBuffer.reset(); + return; + } + payloadLength = (recvBuffer.get() & 0xFF) << 8 | (recvBuffer.get() & 0xFF); + } else { + if (recvBuffer.remaining() < 8) { + recvBuffer.reset(); + return; + } + payloadLength = recvBuffer.getLong(); + } + + int maskKeySize = isMasked ? 4 : 0; + if (recvBuffer.remaining() < maskKeySize + payloadLength) { + recvBuffer.reset(); + return; + } + + byte[] maskKey = null; + if (isMasked) { + maskKey = new byte[4]; + recvBuffer.get(maskKey); + } + + byte[] payload = new byte[(int) payloadLength]; + recvBuffer.get(payload); + + if (isMasked) { + for (int i = 0; i < payload.length; i++) { + payload[i] ^= maskKey[i & 3]; + } + } + + switch (opcode) { + case WebSocketOpcode.BINARY -> handler.onBinaryMessage(this, payload); + case WebSocketOpcode.PING -> { + try { + writeFrame(WebSocketOpcode.PONG, payload, payload.length); + } catch (IOException e) { + LOG.error("Failed to send pong", e); + } + } + case WebSocketOpcode.CLOSE -> { + int code = WebSocketCloseCode.NORMAL_CLOSURE; + if (payload.length >= 2) { + code = ((payload[0] & 0xFF) << 8) | (payload[1] & 0xFF); + } + try { + sendClose(code, null); + } catch (IOException e) { + // client may have already disconnected + } + ClientHandler.this.running.set(false); + isClosed = true; + } + } + } + + recvBuffer.compact(); + recvBuffer.flip(); + } + + private boolean performHandshake() throws IOException { + StringBuilder request = new StringBuilder(); + byte[] buf = new byte[1]; + while (true) { + int read = in.read(buf); + if (read <= 0) { + return false; + } + request.append((char) buf[0]); + if (request.toString().endsWith("\r\n\r\n")) { + break; + } + if (request.length() > 8192) { + return false; + } + } + + String key = null; + for (String line : request.toString().split("\r\n")) { + if (line.toLowerCase().startsWith("sec-websocket-key:")) { + key = line.substring(18).trim(); + break; + } + } + + if (key == null) { + return false; + } + + String acceptKey = computeAcceptKey(key); + + String response = "HTTP/1.1 101 Switching Protocols\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Accept: " + acceptKey + "\r\n" + + "\r\n"; + out.write(response.getBytes(StandardCharsets.US_ASCII)); + out.flush(); + + return true; + } + + private synchronized void writeFrame(int opcode, byte[] payload, int length) throws IOException { + // first byte: FIN + opcode + out.write(0x80 | (opcode & 0x0F)); + + // payload length (unmasked - server to client) + if (length <= 125) { + out.write(length); + } else if (length <= 65_535) { + out.write(126); + out.write((length >> 8) & 0xFF); + out.write(length & 0xFF); + } else { + out.write(127); + ByteBuffer lenBuf = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN); + lenBuf.putLong(length); + out.write(lenBuf.array()); + } + + // payload + out.write(payload, 0, length); + out.flush(); + } + + void start() { + if (running.getAndSet(true)) { + return; + } + + readThread = new Thread(() -> { + try { + socket.setSoTimeout(100); + + in = socket.getInputStream(); + out = socket.getOutputStream(); + + if (!performHandshake()) { + LOG.error("Handshake failed"); + return; + } + + byte[] readBuf = new byte[8192]; + + while (running.get() && !isClosed) { + int read; + try { + read = in.read(readBuf); + } catch (SocketTimeoutException e) { + continue; + } + if (read <= 0) { + break; + } + + // append to recvBuffer + recvBuffer.compact(); + if (recvBuffer.remaining() < read) { + // should not happen with 64k buffer in tests + LOG.error("Receive buffer overflow"); + break; + } + recvBuffer.put(readBuf, 0, read); + recvBuffer.flip(); + + handleRead(); + } + } catch (IOException e) { + if (running.get()) { + LOG.error("Client error", e); + } + } + }, "WebSocket-Client-" + socket.getPort()); + readThread.start(); + } + } +} From 755ff35c93d856581e580fd08202369f9eb23f4b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 12 Mar 2026 15:40:42 +0100 Subject: [PATCH 186/230] Move tests that need running server to core module --- .../cutlass/line/AbstractLineSenderTest.java | 8 - .../line/tcp/AbstractLineTcpSenderTest.java | 73 - .../line/tcp/LineTcpAuthSenderTest.java | 151 -- .../cutlass/line/tcp/LineTcpSenderTest.java | 1435 +---------------- .../line/udp/AbstractLineUdpSenderTest.java | 118 -- .../cutlass/line/udp/LineUdpSenderTest.java | 212 --- 6 files changed, 28 insertions(+), 1969 deletions(-) delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/tcp/AbstractLineTcpSenderTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpAuthSenderTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/udp/AbstractLineUdpSenderTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/udp/LineUdpSenderTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java index 9180b21..323b6c8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java @@ -2,8 +2,6 @@ import io.questdb.client.cutlass.line.AbstractLineSender; import io.questdb.client.test.AbstractQdbTest; -import org.junit.Assume; -import org.junit.BeforeClass; import java.lang.reflect.Array; @@ -16,12 +14,6 @@ public static T createDoubleArray(int... shape) { return buildNestedArray(ArrayDataType.DOUBLE, shape, 0, indices); } - @BeforeClass - public static void setUpStatic() { - AbstractQdbTest.setUpStatic(); - Assume.assumeTrue(getQuestDBRunning()); - } - @SuppressWarnings("unchecked") private static T buildNestedArray(ArrayDataType dataType, int[] shape, int currentDim, int[] indices) { if (currentDim == shape.length - 1) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/AbstractLineTcpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/AbstractLineTcpSenderTest.java deleted file mode 100644 index 6493c5e..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/AbstractLineTcpSenderTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.tcp; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.auth.AuthUtils; -import io.questdb.client.std.Numbers; -import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; - -import java.security.PrivateKey; -import java.util.function.Consumer; - -/** - * Base class for TCP sender integration tests. - * Provides helper methods for creating TCP senders and managing test tables. - */ -public abstract class AbstractLineTcpSenderTest extends AbstractLineSenderTest { - protected static final String AUTH_KEY_ID1 = "testUser1"; - protected final static String AUTH_KEY_ID2_INVALID = "invalid"; - protected final static int HOST = Numbers.parseIPv4("127.0.0.1"); - protected static final Consumer SET_TABLE_NAME_ACTION = s -> s.table("test_mytable"); - protected final static String TOKEN = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; - protected final static PrivateKey AUTH_PRIVATE_KEY1 = AuthUtils.toPrivateKey(TOKEN); - - /** - * Get whether the ILP TCP protocol is authenticated. - */ - protected static boolean getIlpTcpAuthEnabled() { - return getConfigBool("QUESTDB_ILP_TCP_AUTH_ENABLE", "questdb.ilp.tcp.auth.enable", false); - } - - /** - * Get whether the ILP TCP protocol is secure (TLS). - */ - protected static boolean getIlpTcpTlsEnabled() { - return getConfigBool("QUESTDB_ILP_TCP_TLS_ENABLE", "questdb.ilp.tcp.tls.enable", false); - } - - /** - * Create a TCP sender with specified protocol version. - * - * @param protocolVersion the ILP protocol version (V1, V2, or V3) - */ - protected Sender createTcpSender(int protocolVersion) { - return Sender.builder(Sender.Transport.TCP) - .address(getQuestDbHost()) - .port(getIlpTcpPort()) - .protocolVersion(protocolVersion) - .build(); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpAuthSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpAuthSenderTest.java deleted file mode 100644 index 2e44511..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpAuthSenderTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.tcp; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.AbstractLineTcpSender; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.line.LineTcpSenderV2; -import io.questdb.client.std.datetime.microtime.Micros; -import org.junit.Assume; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.time.temporal.ChronoUnit; - -import static io.questdb.client.Sender.PROTOCOL_VERSION_V2; -import static io.questdb.client.Sender.Transport; -import static io.questdb.client.test.tools.TestUtils.assertContains; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.fail; - -/** - * Tests for LineTcpSender. - *

      - * Unit tests use DummyLineChannel/ByteChannel (no server needed). - * Integration tests use external QuestDB via AbstractLineTcpSenderTest - * infrastructure. - */ -public class LineTcpAuthSenderTest extends AbstractLineTcpSenderTest { - @BeforeClass - public static void setUpStatic() { - AbstractLineTcpSenderTest.setUpStatic(); - Assume.assumeTrue(getIlpTcpAuthEnabled()); - } - - @Test - public void testAuthSuccess() throws Exception { - useTable("test_auth_success"); - - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), 256 * 1024)) { - sender.authenticate(AUTH_KEY_ID1, AUTH_PRIVATE_KEY1); - sender.metric("test_auth_success").field("my int field", 42).$(); - sender.flush(); - } - - assertTableExistsEventually("test_auth_success"); - } - - @Test - public void testAuthWrongKey() { - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), 2048)) { - sender.authenticate(AUTH_KEY_ID2_INVALID, AUTH_PRIVATE_KEY1); - // 30 seconds should be enough even on a slow CI server - long deadline = System.nanoTime() + SECONDS.toNanos(30); - while (System.nanoTime() < deadline) { - sender.metric("test_auth_wrong_key").field("my int field", 42).$(); - sender.flush(); - } - fail("Client fail to detected qdb server closed a connection due to wrong credentials"); - } catch (LineSenderException expected) { - // ignored - } - } - - @Test - public void testBuilderAuthSuccess() throws Exception { - useTable("test_builder_auth_success"); - - try (Sender sender = Sender.builder(Transport.TCP) - .address("127.0.0.1:" + getIlpTcpPort()) - .enableAuth(AUTH_KEY_ID1).authToken(TOKEN) - .protocolVersion(PROTOCOL_VERSION_V2) - .build()) { - sender.table("test_builder_auth_success").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_auth_success"); - } - - @Test - public void testBuilderAuthSuccess_confString() throws Exception { - useTable("test_builder_auth_success_conf_string"); - - try (Sender sender = Sender.fromConfig("tcp::addr=127.0.0.1:" + getIlpTcpPort() + ";user=" + AUTH_KEY_ID1 - + ";token=" + TOKEN + ";protocol_version=2;")) { - sender.table("test_builder_auth_success_conf_string").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_auth_success_conf_string"); - } - - @Test - public void testConfString() throws Exception { - useTable("test_conf_string"); - - String confString = "tcp::addr=127.0.0.1:" + getIlpTcpPort() + ";user=" + AUTH_KEY_ID1 + ";token=" + TOKEN - + ";protocol_version=2;"; - try (Sender sender = Sender.fromConfig(confString)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table("test_conf_string") - .longColumn("int_field", 42) - .boolColumn("bool_field", true) - .stringColumn("string_field", "foo") - .doubleColumn("double_field", 42.0) - .timestampColumn("ts_field", tsMicros, ChronoUnit.MICROS) - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_conf_string", 1); - assertSqlEventually( - "int_field\tbool_field\tstring_field\tdouble_field\tts_field\ttimestamp\n" + - "42\ttrue\tfoo\t42.0\t2022-02-25T00:00:00.000000000Z\t2022-02-25T00:00:00.000000000Z\n", - "select int_field, bool_field, string_field, double_field, ts_field, timestamp from test_conf_string"); - } - - @Test - public void testMinBufferSizeWhenAuth() { - int tinyCapacity = 42; - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), tinyCapacity)) { - sender.authenticate(AUTH_KEY_ID1, AUTH_PRIVATE_KEY1); - fail(); - } catch (LineSenderException e) { - assertContains(e.getMessage(), "challenge did not fit into buffer"); - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java index bc4152a..00c7241 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/LineTcpSenderTest.java @@ -25,155 +25,28 @@ package io.questdb.client.test.cutlass.line.tcp; import io.questdb.client.Sender; -import io.questdb.client.cairo.ColumnType; -import io.questdb.client.cutlass.line.AbstractLineTcpSender; import io.questdb.client.cutlass.line.LineChannel; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.LineTcpSenderV2; -import io.questdb.client.cutlass.line.array.DoubleArray; -import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; -import io.questdb.client.network.NetworkFacadeImpl; -import io.questdb.client.std.Decimal256; -import io.questdb.client.std.datetime.microtime.Micros; +import io.questdb.client.cutlass.line.LineTcpSenderV3; import io.questdb.client.std.datetime.microtime.MicrosecondClockImpl; -import io.questdb.client.std.datetime.nanotime.Nanos; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.BeforeClass; +import io.questdb.client.test.AbstractTest; import org.junit.Test; -import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Random; import java.util.function.Consumer; -import static io.questdb.client.Sender.*; import static io.questdb.client.test.tools.TestUtils.assertContains; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.*; /** - * Tests for LineTcpSender. + * Unit tests for LineTcpSender. *

      - * Unit tests use DummyLineChannel/ByteChannel (no server needed). - * Integration tests use external QuestDB via AbstractLineTcpSenderTest - * infrastructure. + * These tests use DummyLineChannel/ByteChannel (no server needed). + * Integration tests have been migrated to core module's LineTcpBootstrapTest. */ -public class LineTcpSenderTest extends AbstractLineTcpSenderTest { - @BeforeClass - public static void setUpStatic() { - AbstractLineTcpSenderTest.setUpStatic(); - Assume.assumeFalse(getIlpTcpAuthEnabled()); - } - - @Test - public void testArrayAtNow() throws Exception { - String table = "test_array_at_now"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2); - DoubleArray a1 = new DoubleArray(1, 1, 2, 1).setAll(1)) { - sender.table(table) - .symbol("x", "42i") - .symbol("y", "[6f1.0,2.5,3.0,4.5,5.0]") // ensuring no array parsing for symbol - .longColumn("l1", 23452345) - .doubleArray("a1", a1) - .atNow(); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - } - - @Test - public void testArrayDouble() throws Exception { - String table = "test_array_double"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2); - DoubleArray a4 = new DoubleArray(1, 1, 2, 1).setAll(4); - DoubleArray a5 = new DoubleArray(3, 2, 1, 4, 1).setAll(5); - DoubleArray a6 = new DoubleArray(1, 3, 4, 2, 1, 1).setAll(6)) { - long ts = Micros.floor("2025-02-22T00:00:00.000000000Z"); - double[] arr1d = createDoubleArray(5); - double[][] arr2d = createDoubleArray(2, 3); - double[][][] arr3d = createDoubleArray(1, 2, 3); - sender.table(table) - .symbol("x", "42i") - .symbol("y", "[6f1.0,2.5,3.0,4.5,5.0]") // ensuring no array parsing for symbol - .longColumn("l1", 23452345) - .doubleArray("a1", arr1d) - .doubleArray("a2", arr2d) - .doubleArray("a3", arr3d) - .doubleArray("a4", a4) - .doubleArray("a5", a5) - .doubleArray("a6", a6) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - } - - @Test - public void testAuthWrongKey() throws Exception { - try (AbstractLineTcpSender sender = LineTcpSenderV2.newSender(HOST, getIlpTcpPort(), 2048)) { - sender.authenticate(AUTH_KEY_ID2_INVALID, AUTH_PRIVATE_KEY1); - // 30 seconds should be enough even on a slow CI server - long deadline = System.nanoTime() + SECONDS.toNanos(30); - while (System.nanoTime() < deadline) { - sender.metric("test_auth_wrong_key").field("my int field", 42).$(); - sender.flush(); - } - fail("Client fail to detected qdb server closed a connection due to wrong credentials"); - } catch (LineSenderException expected) { - // ignored - } - } - - @Test - public void testBuilderPlainText_addressWithExplicitIpAndPort() throws Exception { - useTable("test_builder_plain_text_explicit_ip_port"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_builder_plain_text_explicit_ip_port").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_plain_text_explicit_ip_port"); - } - - @Test - public void testBuilderPlainText_addressWithHostnameAndPort() throws Exception { - useTable("test_builder_plain_text_hostname_port"); - - try (Sender sender = Sender.builder(Sender.Transport.TCP) - .address("localhost:" + getIlpTcpPort()) - .protocolVersion(PROTOCOL_VERSION_V2) - .build()) { - sender.table("test_builder_plain_text_hostname_port").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_plain_text_hostname_port"); - } - - @Test - public void testBuilderPlainText_addressWithIpAndPort() throws Exception { - useTable("test_builder_plain_text_ip_port"); - - String address = "127.0.0.1:" + getIlpTcpPort(); - try (Sender sender = Sender.builder(Sender.Transport.TCP) - .address(address) - .protocolVersion(PROTOCOL_VERSION_V2) - .build()) { - sender.table("test_builder_plain_text_ip_port").longColumn("my int field", 42).atNow(); - sender.flush(); - } - - assertTableExistsEventually("test_builder_plain_text_ip_port"); - } +public class LineTcpSenderTest extends AbstractTest { + protected static final Consumer SET_TABLE_NAME_ACTION = s -> s.table("test_mytable"); @Test public void testCannotStartNewRowBeforeClosingTheExistingAfterValidationError() { @@ -199,45 +72,12 @@ public void testCannotStartNewRowBeforeClosingTheExistingAfterValidationError() @Test public void testCloseIdempotent() { DummyLineChannel channel = new DummyLineChannel(); - AbstractLineTcpSender sender = new LineTcpSenderV2(channel, 1000, 127); + LineTcpSenderV2 sender = new LineTcpSenderV2(channel, 1000, 127); sender.close(); sender.close(); assertEquals(1, channel.closeCounter); } - @Test - public void testCloseImpliesFlush() throws Exception { - useTable("test_close_implies_flush"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_close_implies_flush").longColumn("my int field", 42).atNow(); - } - - assertTableExistsEventually("test_close_implies_flush"); - } - - @Test - public void testConfString_autoFlushBytes() throws Exception { - useTable("test_conf_string_auto_flush_bytes"); - - String confString = "tcp::addr=localhost:" + getIlpTcpPort() + ";auto_flush_bytes=1;protocol_version=2;"; // the - // minimal - // allowed - // buffer - // size - try (Sender sender = Sender.fromConfig(confString)) { - // just 2 rows must be enough to trigger flush - // why not 1? the first byte of the 2nd row will flush the last byte of the 1st - // row - sender.table("test_conf_string_auto_flush_bytes").longColumn("my int field", 42).atNow(); - sender.table("test_conf_string_auto_flush_bytes").longColumn("my int field", 42).atNow(); - - // make sure to assert before closing the Sender - // since the Sender will always flush on close - assertTableExistsEventually("test_conf_string_auto_flush_bytes"); - } - } - @Test public void testControlCharInColumnName() { assertControlCharacterException(); @@ -248,1049 +88,68 @@ public void testControlCharInTableName() { assertControlCharacterException(); } - @Test - public void testCreateTimestampColumnsWithDesignatedInstantV1() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123123123Z"), null, PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123123000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedInstantV2() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123123123Z"), null, PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123123000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMicrosV1() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z"), ChronoUnit.MICROS, - PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMicrosV2() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z"), ChronoUnit.MICROS, - PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMillisV1() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z") / 1000, ChronoUnit.MILLIS, - PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123000000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedMillisV2() throws Exception { - testCreateTimestampColumns(Micros.floor("2025-11-20T10:55:24.123456000Z") / 1000, ChronoUnit.MILLIS, - PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123000000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedNanosV1() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123456789Z"), ChronoUnit.NANOS, PROTOCOL_VERSION_V1, - new int[]{ColumnType.TIMESTAMP, ColumnType.TIMESTAMP, ColumnType.TIMESTAMP}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testCreateTimestampColumnsWithDesignatedNanosV2() throws Exception { - testCreateTimestampColumns(Nanos.floor("2025-11-20T10:55:24.123456789Z"), ChronoUnit.NANOS, PROTOCOL_VERSION_V2, - new int[]{ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO, ColumnType.TIMESTAMP_NANO}, - "1.111\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-19T10:55:24.123000000Z\t2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.123456000Z"); - } - - @Test - public void testDecimalDefaultValuesWithoutWal() throws Exception { - useTable("test_decimal_default_values_without_wal"); - execute( - "CREATE TABLE test_decimal_default_values_without_wal (\n" + - " dec8 DECIMAL(2, 0),\n" + - " dec16 DECIMAL(4, 1),\n" + - " dec32 DECIMAL(8, 2),\n" + - " dec64 DECIMAL(16, 4),\n" + - " dec128 DECIMAL(34, 8),\n" + - " dec256 DECIMAL(64, 16),\n" + - " value INT,\n" + - " ts TIMESTAMP\n" + - ") TIMESTAMP(ts) PARTITION BY DAY BYPASS WAL\n"); - - assertTableExistsEventually("test_decimal_default_values_without_wal"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - sender.table("test_decimal_default_values_without_wal") - .longColumn("value", 1) - .at(100_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_decimal_default_values_without_wal", 1); - assertSqlEventually( - "dec8\tdec16\tdec32\tdec64\tdec128\tdec256\tvalue\tts\n" + - "null\tnull\tnull\tnull\tnull\tnull\t1\t1970-01-01T00:00:00.100000000Z\n", - "select dec8, dec16, dec32, dec64, dec128, dec256, value, ts from test_decimal_default_values_without_wal"); - } - - @Test - public void testDouble_edgeValues() throws Exception { - useTable("test_double_edge_values"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2022-02-25"); - sender.table("test_double_edge_values") - .doubleColumn("negative_inf", Double.NEGATIVE_INFINITY) - .doubleColumn("positive_inf", Double.POSITIVE_INFINITY) - .doubleColumn("nan", Double.NaN) - .doubleColumn("max_value", Double.MAX_VALUE) - .doubleColumn("min_value", Double.MIN_VALUE) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_double_edge_values", 1); - assertSqlEventually( - "negative_inf\tpositive_inf\tnan\tmax_value\tmin_value\ttimestamp\n" + - "null\tnull\tnull\t1.7976931348623157E308\t4.9E-324\t2022-02-25T00:00:00.000000000Z\n", - "select negative_inf, positive_inf, nan, max_value, min_value, timestamp from test_double_edge_values"); - } - - @Test - public void testExplicitTimestampColumnIndexIsCleared() throws Exception { - useTable("test_explicit_ts_col_idx_cleared_poison"); - useTable("test_explicit_ts_col_idx_cleared_victim"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2022-02-25"); - // the poison table sets the timestamp column index explicitly - sender.table("test_explicit_ts_col_idx_cleared_poison") - .stringColumn("str_col1", "str_col1") - .stringColumn("str_col2", "str_col2") - .stringColumn("str_col3", "str_col3") - .stringColumn("str_col4", "str_col4") - .timestampColumn("timestamp", ts, ChronoUnit.MICROS) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - assertTableSizeEventually("test_explicit_ts_col_idx_cleared_poison", 1); - - // the victim table does not set the timestamp column index explicitly - sender.table("test_explicit_ts_col_idx_cleared_victim") - .stringColumn("str_col1", "str_col1") - .at(ts, ChronoUnit.MICROS); - sender.flush(); - assertTableSizeEventually("test_explicit_ts_col_idx_cleared_victim", 1); - } - } - - @Test - public void testInsertBadStringIntoUuidColumn() throws Exception { - testValueCannotBeInsertedToUuidColumn("test_insert_bad_string_into_uuid_column", "totally not a uuid"); - } - - @Test - public void testInsertBinaryToOtherColumns() throws Exception { - useTable("test_insert_binary_to_other_columns"); - execute( - "CREATE TABLE test_insert_binary_to_other_columns (\n" + - " x SYMBOL,\n" + - " y VARCHAR,\n" + - " a1 DOUBLE,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_binary_to_other_columns"); - - // send text double to symbol column - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V1)) { - sender.table("test_insert_binary_to_other_columns") - .doubleColumn("x", 9999.0) - .stringColumn("y", "ystr") - .doubleColumn("a1", 1) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - } - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - // insert binary double to symbol column - sender.table("test_insert_binary_to_other_columns") - .doubleColumn("x", 10000.0) - .stringColumn("y", "ystr") - .doubleColumn("a1", 1) - .at(100000000001L, ChronoUnit.MICROS); - sender.flush(); - - // insert binary double to string column (should be rejected) - sender.table("test_insert_binary_to_other_columns") - .symbol("x", "x1") - .doubleColumn("y", 9999.0) - .doubleColumn("a1", 1) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - // insert string to double column (should be rejected) - sender.table("test_insert_binary_to_other_columns") - .symbol("x", "x1") - .stringColumn("y", "ystr") - .stringColumn("a1", "11.u") - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - // insert array column to double (should be rejected) - sender.table("test_insert_binary_to_other_columns") - .symbol("x", "x1") - .stringColumn("y", "ystr") - .doubleArray("a1", new double[]{1.0, 2.0}) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_binary_to_other_columns", 2); - assertSqlEventually( - "x\ty\ta1\ttimestamp\n" + - "9999.0\tystr\t1.0\t1970-01-02T03:46:40.000000000Z\n" + - "10000.0\tystr\t1.0\t1970-01-02T03:46:40.000001000Z\n", - "select x, y, a1, timestamp from test_insert_binary_to_other_columns order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatBasic() throws Exception { - String tableName = "test_decimal_text_format_basic"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_basic (\n" + - " price DECIMAL(10, 2),\n" + - " quantity DECIMAL(15, 4),\n" + - " rate DECIMAL(8, 5),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Basic positive decimal - sender.table(tableName) - .decimalColumn("price", "123.45") - .decimalColumn("quantity", "100.0000") - .decimalColumn("rate", "0.12345") - .at(100000000000L, ChronoUnit.MICROS); - - // Negative decimal - sender.table(tableName) - .decimalColumn("price", "-45.67") - .decimalColumn("quantity", "-10.5000") - .decimalColumn("rate", "-0.00001") - .at(100000000001L, ChronoUnit.MICROS); - - // Small values - sender.table(tableName) - .decimalColumn("price", "0.01") - .decimalColumn("quantity", "0.0001") - .decimalColumn("rate", "0.00000") - .at(100000000002L, ChronoUnit.MICROS); - - // Integer strings (no decimal point) - sender.table(tableName) - .decimalColumn("price", "999") - .decimalColumn("quantity", "42") - .decimalColumn("rate", "1") - .at(100000000003L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 4); - assertSqlEventually( - "price\tquantity\trate\ttimestamp\n" + - "123.45\t100.0000\t0.12345\t1970-01-02T03:46:40.000000000Z\n" + - "-45.67\t-10.5000\t-0.00001\t1970-01-02T03:46:40.000001000Z\n" + - "0.01\t0.0001\t0.00000\t1970-01-02T03:46:40.000002000Z\n" + - "999.00\t42.0000\t1.00000\t1970-01-02T03:46:40.000003000Z\n", - "select price, quantity, rate, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatEdgeCases() throws Exception { - String tableName = "test_decimal_text_format_edge_cases"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_edge_cases (\n" + - " value DECIMAL(20, 10),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Explicit positive sign - sender.table(tableName) - .decimalColumn("value", "+123.456") - .at(100000000000L, ChronoUnit.MICROS); - - // Leading zeros - sender.table(tableName) - .decimalColumn("value", "000123.450000") - .at(100000000001L, ChronoUnit.MICROS); - - // Very small value - sender.table(tableName) - .decimalColumn("value", "0.0000000001") - .at(100000000002L, ChronoUnit.MICROS); - - // Zero with decimal point - sender.table(tableName) - .decimalColumn("value", "0.0") - .at(100000000003L, ChronoUnit.MICROS); - - // Just zero - sender.table(tableName) - .decimalColumn("value", "0") - .at(100000000004L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 5); - assertSqlEventually( - "value\ttimestamp\n" + - "123.4560000000\t1970-01-02T03:46:40.000000000Z\n" + - "123.4500000000\t1970-01-02T03:46:40.000001000Z\n" + - "0.0000000001\t1970-01-02T03:46:40.000002000Z\n" + - "0.0000000000\t1970-01-02T03:46:40.000003000Z\n" + - "0.0000000000\t1970-01-02T03:46:40.000004000Z\n", - "select value, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatEquivalence() throws Exception { - String tableName = "test_decimal_text_format_equivalence"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_equivalence (\n" + - " text_format DECIMAL(10, 3),\n" + - " binary_format DECIMAL(10, 3),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Test various values sent via both text and binary formats - sender.table(tableName) - .decimalColumn("text_format", "123.450") - .decimalColumn("binary_format", Decimal256.fromLong(123450, 3)) - .at(100000000000L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("text_format", "-45.670") - .decimalColumn("binary_format", Decimal256.fromLong(-45670, 3)) - .at(100000000001L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("text_format", "0.001") - .decimalColumn("binary_format", Decimal256.fromLong(1, 3)) - .at(100000000002L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 3); - assertSqlEventually( - "text_format\tbinary_format\ttimestamp\n" + - "123.450\t123.450\t1970-01-02T03:46:40.000000000Z\n" + - "-45.670\t-45.670\t1970-01-02T03:46:40.000001000Z\n" + - "0.001\t0.001\t1970-01-02T03:46:40.000002000Z\n", - "select text_format, binary_format, timestamp from " + tableName + " order by timestamp"); - } - @Test public void testInsertDecimalTextFormatInvalid() throws Exception { - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { + try (Sender sender = new LineTcpSenderV3(new DummyLineChannel(), 4096, 127)) { sender.table("test"); // Test invalid characters try { sender.decimalColumn("value", "abc"); - Assert.fail("Letters should throw exception"); + fail("Letters should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test multiple dots try { sender.decimalColumn("value", "12.34.56"); - Assert.fail("Multiple dots should throw exception"); + fail("Multiple dots should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test multiple signs try { sender.decimalColumn("value", "+-123"); - Assert.fail("Multiple signs should throw exception"); + fail("Multiple signs should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test special characters try { sender.decimalColumn("value", "12$34"); - Assert.fail("Special characters should throw exception"); + fail("Special characters should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test empty decimal try { sender.decimalColumn("value", ""); - Assert.fail("Empty string should throw exception"); + fail("Empty string should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test invalid exponent try { sender.decimalColumn("value", "1.23eABC"); - Assert.fail("Invalid exponent should throw exception"); + fail("Invalid exponent should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } // Test incomplete exponent try { sender.decimalColumn("value", "1.23e"); - Assert.fail("Incomplete exponent should throw exception"); + fail("Incomplete exponent should throw exception"); } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "Failed to parse sent decimal value"); + assertContains(e.getMessage(), "Failed to parse sent decimal value"); } } } - @Test - public void testInsertDecimalTextFormatPrecisionOverflow() throws Exception { - String tableName = "test_decimal_text_format_precision_overflow"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_precision_overflow (\n" + - " x DECIMAL(6, 3),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Value that exceeds column precision (6 digits total, 3 after decimal) - // 1000.000 has 7 digits precision, should be rejected - sender.table(tableName) - .decimalColumn("x", "1000.000") - .at(100000000000L, ChronoUnit.MICROS); - - // Another value that exceeds precision - sender.table(tableName) - .decimalColumn("x", "12345.678") - .at(100000000001L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertSqlEventually( - "x\ttimestamp\n", - "select x, timestamp from " + tableName); - } - - @Test - public void testInsertDecimalTextFormatScientificNotation() throws Exception { - String tableName = "test_decimal_text_format_scientific_notation"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_scientific_notation (\n" + - " large DECIMAL(15, 2),\n" + - " small DECIMAL(20, 15),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Scientific notation with positive exponent - sender.table(tableName) - .decimalColumn("large", "1.23e5") - .decimalColumn("small", "1.23e-10") - .at(100000000000L, ChronoUnit.MICROS); - - // Scientific notation with uppercase E - sender.table(tableName) - .decimalColumn("large", "4.56E3") - .decimalColumn("small", "4.56E-8") - .at(100000000001L, ChronoUnit.MICROS); - - // Negative value with scientific notation - sender.table(tableName) - .decimalColumn("large", "-9.99e2") - .decimalColumn("small", "-1.5e-12") - .at(100000000002L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 3); - assertSqlEventually( - "large\tsmall\ttimestamp\n" + - "123000.00\t0.000000000123000\t1970-01-02T03:46:40.000000000Z\n" + - "4560.00\t0.000000045600000\t1970-01-02T03:46:40.000001000Z\n" + - "-999.00\t-0.000000000001500\t1970-01-02T03:46:40.000002000Z\n", - "select large, small, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimalTextFormatTrailingZeros() throws Exception { - String tableName = "test_decimal_text_format_trailing_zeros"; - useTable(tableName); - execute( - "CREATE TABLE test_decimal_text_format_trailing_zeros (\n" + - " value1 DECIMAL(10, 3),\n" + - " value2 DECIMAL(12, 5),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Trailing zeros should be preserved in scale - sender.table(tableName) - .decimalColumn("value1", "100.000") - .decimalColumn("value2", "50.00000") - .at(100000000000L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("value1", "1.200") - .decimalColumn("value2", "0.12300") - .at(100000000001L, ChronoUnit.MICROS); - - sender.table(tableName) - .decimalColumn("value1", "0.100") - .decimalColumn("value2", "0.00100") - .at(100000000002L, ChronoUnit.MICROS); - - sender.flush(); - } - - assertTableSizeEventually(tableName, 3); - assertSqlEventually( - "value1\tvalue2\ttimestamp\n" + - "100.000\t50.00000\t1970-01-02T03:46:40.000000000Z\n" + - "1.200\t0.12300\t1970-01-02T03:46:40.000001000Z\n" + - "0.100\t0.00100\t1970-01-02T03:46:40.000002000Z\n", - "select value1, value2, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertDecimals() throws Exception { - String tableName = "test_insert_decimals"; - useTable(tableName); - execute( - "CREATE TABLE test_insert_decimals (\n" + - " a DECIMAL(9, 0),\n" + - " b DECIMAL(9, 3),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - sender.table(tableName) - .decimalColumn("a", Decimal256.fromLong(12345, 0)) - .decimalColumn("b", Decimal256.fromLong(12345, 2)) - .at(100000000000L, ChronoUnit.MICROS); - - // Decimal without rescale - sender.table(tableName) - .decimalColumn("a", Decimal256.NULL_VALUE) - .decimalColumn("b", Decimal256.fromLong(123456, 3)) - .at(100000000001L, ChronoUnit.MICROS); - - // Integers -> Decimal - sender.table(tableName) - .longColumn("a", 42) - .longColumn("b", 42) - .at(100000000002L, ChronoUnit.MICROS); - - // Strings -> Decimal without rescale - sender.table(tableName) - .stringColumn("a", "42") - .stringColumn("b", "42.123") - .at(100000000003L, ChronoUnit.MICROS); - - // Strings -> Decimal with rescale - sender.table(tableName) - .stringColumn("a", "42.0") - .stringColumn("b", "42.1") - .at(100000000004L, ChronoUnit.MICROS); - - // Doubles -> Decimal - sender.table(tableName) - .doubleColumn("a", 42d) - .doubleColumn("b", 42.1d) - .at(100000000005L, ChronoUnit.MICROS); - - // NaN/Inf Doubles -> Decimal - sender.table(tableName) - .doubleColumn("a", Double.NaN) - .doubleColumn("b", Double.POSITIVE_INFINITY) - .at(100000000006L, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(tableName, 7); - assertSqlEventually( - "a\tb\ttimestamp\n" + - "12345\t123.450\t1970-01-02T03:46:40.000000000Z\n" + - "null\t123.456\t1970-01-02T03:46:40.000001000Z\n" + - "42\t42.000\t1970-01-02T03:46:40.000002000Z\n" + - "42\t42.123\t1970-01-02T03:46:40.000003000Z\n" + - "42\t42.100\t1970-01-02T03:46:40.000004000Z\n" + - "42\t42.100\t1970-01-02T03:46:40.000005000Z\n" + - "null\tnull\t1970-01-02T03:46:40.000006000Z\n", - "select a, b, timestamp from " + tableName + " order by timestamp"); - } - - @Test - public void testInsertInvalidDecimals() throws Exception { - String tableName = "test_invalid_decimal_test"; - useTable(tableName); - execute( - "CREATE TABLE test_invalid_decimal_test (\n" + - " x DECIMAL(6, 3),\n" + - " y DECIMAL(76, 73),\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V3)) { - // Integers out of bound (with scaling, 1234 becomes 1234.000 which have a - // precision of 7). - sender.table(tableName) - .longColumn("x", 1234) - .at(100000000000L, ChronoUnit.MICROS); - - // Integers overbound during the rescale process. - sender.table(tableName) - .longColumn("y", 12345) - .at(100000000001L, ChronoUnit.MICROS); - - // Floating points with a scale greater than expected. - sender.table(tableName) - .doubleColumn("x", 1.2345d) - .at(100000000002L, ChronoUnit.MICROS); - - // Floating points with a precision greater than expected. - sender.table(tableName) - .doubleColumn("x", 12345.678d) - .at(100000000003L, ChronoUnit.MICROS); - - // String that is not a valid decimal. - sender.table(tableName) - .stringColumn("x", "abc") - .at(100000000004L, ChronoUnit.MICROS); - - // String that has a too big precision. - sender.table(tableName) - .stringColumn("x", "1E8") - .at(100000000005L, ChronoUnit.MICROS); - - // Decimal with a too big precision. - sender.table(tableName) - .decimalColumn("x", Decimal256.fromLong(12345678, 3)) - .at(100000000006L, ChronoUnit.MICROS); - - // Decimal with a too big precision when scaled. - sender.table(tableName) - .decimalColumn("y", Decimal256.fromLong(12345, 0)) - .at(100000000007L, ChronoUnit.MICROS); - sender.flush(); - - // Decimal loosing precision - sender.table(tableName) - .decimalColumn("x", Decimal256.fromLong(123456, 4)) - .at(100000000007L, ChronoUnit.MICROS); - sender.flush(); - } - - assertSqlEventually( - "x\ty\ttimestamp\n", - "select x, y, timestamp from " + tableName); - } - - @Test - public void testInsertLargeArray() throws Exception { - String tableName = "test_arr_large_test"; - useTable(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - double[] arr = createDoubleArray(10_000_000); - sender.table(tableName) - .doubleArray("arr", arr) - .at(100000000000L, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testInsertNonAsciiStringAndUuid() throws Exception { - // this is to check that a non-ASCII string will not prevent - // parsing a subsequent UUID - useTable("test_insert_non_ascii_string_and_uuid"); - execute( - "CREATE TABLE test_insert_non_ascii_string_and_uuid (\n" + - " s STRING,\n" + - " u UUID,\n" + - " ts TIMESTAMP\n" + - ") TIMESTAMP(ts) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_non_ascii_string_and_uuid"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table("test_insert_non_ascii_string_and_uuid") - .stringColumn("s", "non-ascii äöü") - .stringColumn("u", "11111111-2222-3333-4444-555555555555") - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_non_ascii_string_and_uuid", 1); - assertSqlEventually( - "s\tu\tts\n" + - "non-ascii äöü\t11111111-2222-3333-4444-555555555555\t2022-02-25T00:00:00.000000000Z\n", - "select s, u, ts from test_insert_non_ascii_string_and_uuid"); - } - - @Test - public void testInsertNonAsciiStringIntoUuidColumn() throws Exception { - // carefully crafted value so when encoded as UTF-8 it has the same byte length - // as a proper UUID - testValueCannotBeInsertedToUuidColumn("test_insert_non_ascii_string_into_uuid_column", - "11111111-1111-1111-1111-1111111111ü"); - } - - @Test - public void testInsertStringIntoUuidColumn() throws Exception { - useTable("test_insert_string_into_uuid_column"); - execute( - "CREATE TABLE test_insert_string_into_uuid_column (\n" + - " u1 UUID,\n" + - " u2 UUID,\n" + - " u3 UUID,\n" + - " ts TIMESTAMP\n" + - ") TIMESTAMP(ts) PARTITION BY NONE BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_string_into_uuid_column"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table("test_insert_string_into_uuid_column") - .stringColumn("u1", "11111111-1111-1111-1111-111111111111") - // u2 empty -> insert as null - .stringColumn("u3", "33333333-3333-3333-3333-333333333333") - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_string_into_uuid_column", 1); - assertSqlEventually( - "u1\tu3\tts\n" + - "11111111-1111-1111-1111-111111111111\t33333333-3333-3333-3333-333333333333\t2022-02-25T00:00:00.000000000Z\n", - "select u1, u3, ts from test_insert_string_into_uuid_column"); - } - - @Test - public void testInsertTimestampAsInstant() throws Exception { - useTable("test_insert_timestamp_as_instant"); - execute( - "CREATE TABLE test_insert_timestamp_as_instant (\n" + - " ts_col TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_as_instant"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_insert_timestamp_as_instant") - .timestampColumn("ts_col", Instant.parse("2023-02-11T12:30:11.35Z")) - .at(Instant.parse("2022-01-10T20:40:22.54Z")); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_as_instant", 1); - assertSqlEventually( - "ts_col\ttimestamp\n" + - "2023-02-11T12:30:11.350000000Z\t2022-01-10T20:40:22.540000000Z\n", - "select ts_col, timestamp from test_insert_timestamp_as_instant"); - } - - @Test - public void testInsertTimestampMiscUnits() throws Exception { - useTable("test_insert_timestamp_misc_units"); - execute( - "CREATE TABLE test_insert_timestamp_misc_units (\n" + - " unit STRING,\n" + - " ts TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_misc_units"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2023-09-18T12:01:01.01Z"); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "ns") - .timestampColumn("ts", tsMicros * 1000, ChronoUnit.NANOS) - .at(tsMicros * 1000, ChronoUnit.NANOS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "us") - .timestampColumn("ts", tsMicros, ChronoUnit.MICROS) - .at(tsMicros, ChronoUnit.MICROS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "ms") - .timestampColumn("ts", tsMicros / 1000, ChronoUnit.MILLIS) - .at(tsMicros / 1000, ChronoUnit.MILLIS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "s") - .timestampColumn("ts", tsMicros / Micros.SECOND_MICROS, ChronoUnit.SECONDS) - .at(tsMicros / Micros.SECOND_MICROS, ChronoUnit.SECONDS); - sender.table("test_insert_timestamp_misc_units") - .stringColumn("unit", "m") - .timestampColumn("ts", tsMicros / Micros.MINUTE_MICROS, ChronoUnit.MINUTES) - .at(tsMicros / Micros.MINUTE_MICROS, ChronoUnit.MINUTES); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_misc_units", 5); - assertSqlEventually( - "unit\tts\ttimestamp\n" + - "m\t2023-09-18T12:01:00.000000000Z\t2023-09-18T12:01:00.000000000Z\n" + - "s\t2023-09-18T12:01:01.000000000Z\t2023-09-18T12:01:01.000000000Z\n" + - "ns\t2023-09-18T12:01:01.010000000Z\t2023-09-18T12:01:01.010000000Z\n" + - "us\t2023-09-18T12:01:01.010000000Z\t2023-09-18T12:01:01.010000000Z\n" + - "ms\t2023-09-18T12:01:01.010000000Z\t2023-09-18T12:01:01.010000000Z\n", - "select unit, ts, timestamp from test_insert_timestamp_misc_units order by timestamp"); - } - - @Test - public void testInsertTimestampNanoOverflow() throws Exception { - useTable("test_insert_timestamp_nano_overflow"); - execute( - "CREATE TABLE test_insert_timestamp_nano_overflow (\n" + - " ts TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_nano_overflow"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2323-09-18T12:01:01.011568901Z"); - sender.table("test_insert_timestamp_nano_overflow") - .timestampColumn("ts", tsMicros, ChronoUnit.MICROS) - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_nano_overflow", 1); - assertSqlEventually( - "ts\ttimestamp\n" + - "2323-09-18T12:01:01.011568000Z\t2323-09-18T12:01:01.011568000Z\n", - "select ts, timestamp from test_insert_timestamp_nano_overflow"); - } - - @Test - public void testInsertTimestampNanoUnits() throws Exception { - useTable("test_insert_timestamp_nano_units"); - execute( - "CREATE TABLE test_insert_timestamp_nano_units (\n" + - " unit STRING,\n" + - " ts TIMESTAMP,\n" + - " timestamp TIMESTAMP\n" + - ") TIMESTAMP(timestamp) PARTITION BY YEAR BYPASS WAL\n"); - - assertTableExistsEventually("test_insert_timestamp_nano_units"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsNanos = Micros.floor("2023-09-18T12:01:01.011568901Z") * 1000; - sender.table("test_insert_timestamp_nano_units") - .stringColumn("unit", "ns") - .timestampColumn("ts", tsNanos, ChronoUnit.NANOS) - .at(tsNanos, ChronoUnit.NANOS); - sender.flush(); - } - - assertTableSizeEventually("test_insert_timestamp_nano_units", 1); - assertSqlEventually( - "unit\tts\ttimestamp\n" + - "ns\t2023-09-18T12:01:01.011568000Z\t2023-09-18T12:01:01.011568000Z\n", - "select unit, ts, timestamp from test_insert_timestamp_nano_units"); - } - - @Test - public void testMaxNameLength() throws Exception { - PlainTcpLineChannel channel = new PlainTcpLineChannel(NetworkFacadeImpl.INSTANCE, HOST, getIlpTcpPort(), 1024); - try (AbstractLineTcpSender sender = new LineTcpSenderV2(channel, 1024, 20)) { - try { - sender.table("table_with_long______________________name"); - fail(); - } catch (LineSenderException e) { - assertContains(e.getMessage(), - "table name is too long: [name = table_with_long______________________name, maxNameLength=20]"); - } - - try { - sender.table("tab") - .doubleColumn("column_with_long______________________name", 1.0); - fail(); - } catch (LineSenderException e) { - assertContains(e.getMessage(), - "column name is too long: [name = column_with_long______________________name, maxNameLength=20]"); - } - } - } - - @Test - public void testMultipleVarcharCols() throws Exception { - String table = "test_string_table"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2024-02-27"); - sender.table(table) - .stringColumn("string1", "some string") - .stringColumn("string2", "another string") - .stringColumn("string3", "yet another string") - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "string1\tstring2\tstring3\ttimestamp\n" + - "some string\tanother string\tyet another string\t2024-02-27T00:00:00.000000000Z\n", - "select string1, string2, string3, timestamp from " + table); - } - - @Test - public void testServerIgnoresUnfinishedRows() throws Exception { - String tableName = "test_server_ignores_unfinished_rows"; - useTable(tableName); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - // well-formed row first - sender.table(tableName).longColumn("field0", 42) - .longColumn("field1", 42) - .atNow(); - - // failed validation - sender.table(tableName) - .longColumn("field0", 42) - .longColumn("field1\n", 42); - fail("validation should have failed"); - } catch (LineSenderException e) { - // ignored - } - - // make sure the 2nd unfinished row was not inserted by the server - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testSymbolCapacityReload() throws Exception { - // Tests that the client can send many rows with symbols - String tableName = "test_symbol_capacity_table"; - useTable(tableName); - final int N = 1000; - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - Random rnd = new Random(42); - for (int i = 0; i < N; i++) { - sender.table(tableName) - .symbol("sym1", "sym_" + rnd.nextInt(100)) - .symbol("sym2", "s" + rnd.nextInt(10)) - .doubleColumn("dd", rnd.nextDouble()) - .atNow(); - } - sender.flush(); - } - - assertTableSizeEventually(tableName, N); - } - - @Test - public void testSymbolsCannotBeWrittenAfterBool() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.boolColumn("columnName", false)); - } - - @Test - public void testSymbolsCannotBeWrittenAfterDouble() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.doubleColumn("columnName", 42.0)); - } - - @Test - public void testSymbolsCannotBeWrittenAfterLong() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.longColumn("columnName", 42)); - } - - @Test - public void testSymbolsCannotBeWrittenAfterString() throws Exception { - assertSymbolsCannotBeWrittenAfterOtherType(s -> s.stringColumn("columnName", "42")); - } - - @Test - public void testTimestampIngestV1() throws Exception { - testTimestampIngest("test_timestamp_ingest_v1", "TIMESTAMP", PROTOCOL_VERSION_V1, - "ts\tdts\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n", - null); - } - - @Test - public void testTimestampIngestV2() throws Exception { - testTimestampIngest("test_timestamp_ingest_v2", "TIMESTAMP", PROTOCOL_VERSION_V2, - "ts\tdts\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n", - "ts\tdts\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834000000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123456000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2025-11-19T10:55:24.123000000Z\t2025-11-20T10:55:24.834129000Z\n" + - "2300-11-19T10:55:24.123456000Z\t2300-11-20T10:55:24.834129000Z\n"); - } - @Test public void testUnfinishedRowDoesNotContainNewLine() { ByteChannel channel = new ByteChannel(); @@ -1359,71 +218,6 @@ public void testUseAfterClose_tsColumn() { assertExceptionOnClosedSender(SET_TABLE_NAME_ACTION, s -> s.timestampColumn("col", 0, ChronoUnit.MICROS)); } - @Test - public void testUseVarcharAsString() throws Exception { - String table = "test_varchar_string_table"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2024-02-27"); - String expectedValue = "čćžšđçğéíáýůř"; - sender.table(table) - .stringColumn("string1", expectedValue) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "string1\ttimestamp\n" + - "čćžšđçğéíáýůř\t2024-02-27T00:00:00.000000000Z\n", - "select string1, timestamp from " + table); - } - - @Test - public void testWriteAllTypes() throws Exception { - useTable("test_write_all_types"); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2022-02-25"); - sender.table("test_write_all_types") - .longColumn("int_field", 42) - .boolColumn("bool_field", true) - .stringColumn("string_field", "foo") - .doubleColumn("double_field", 42.0) - .timestampColumn("ts_field", ts, ChronoUnit.MICROS) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually("test_write_all_types", 1); - assertSqlEventually( - "int_field\tbool_field\tstring_field\tdouble_field\tts_field\ttimestamp\n" + - "42\ttrue\tfoo\t42.0\t2022-02-25T00:00:00.000000000Z\t2022-02-25T00:00:00.000000000Z\n", - "select int_field, bool_field, string_field, double_field, ts_field, timestamp from test_write_all_types"); - } - - @Test - public void testWriteLongMinMax() throws Exception { - String table = "test_long_min_max_table"; - useTable(table); - - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long ts = Micros.floor("2023-02-22"); - sender.table(table) - .longColumn("max", Long.MAX_VALUE) - .longColumn("min", Long.MIN_VALUE) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "max\tmin\ttimestamp\n" + - "9223372036854775807\tnull\t2023-02-22T00:00:00.000000000Z\n", - "select max, min, timestamp from " + table); - } - private static void assertControlCharacterException() { DummyLineChannel channel = new DummyLineChannel(); try (Sender sender = new LineTcpSenderV2(channel, 1000, 127)) { @@ -1437,6 +231,11 @@ private static void assertControlCharacterException() { } } + private static void assertExceptionOnClosedSender() { + assertExceptionOnClosedSender(s -> { + }, LineTcpSenderTest.SET_TABLE_NAME_ACTION); + } + private static void assertExceptionOnClosedSender(Consumer beforeCloseAction, Consumer afterCloseAction) { DummyLineChannel channel = new DummyLineChannel(); @@ -1451,190 +250,12 @@ private static void assertExceptionOnClosedSender(Consumer beforeCloseAc } } - private static void assertExceptionOnClosedSender() { - assertExceptionOnClosedSender(s -> { - }, LineTcpSenderTest.SET_TABLE_NAME_ACTION); - } - private static void assertNoControlCharacter(CharSequence m) { for (int i = 0, n = m.length(); i < n; i++) { assertFalse(Character.isISOControl(m.charAt(i))); } } - private void assertSymbolsCannotBeWrittenAfterOtherType(Consumer otherTypeWriter) throws Exception { - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - sender.table("test_symbols_cannot_be_written_after_other_type"); - otherTypeWriter.accept(sender); - try { - sender.symbol("name", "value"); - fail("symbols cannot be written after any other column type"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "before any other column types"); - sender.atNow(); - } - } - } - - private void testCreateTimestampColumns(long timestamp, ChronoUnit unit, int protocolVersion, - int[] expectedColumnTypes, String expected) throws Exception { - useTable("test_tab1"); - - try (Sender sender = createTcpSender(protocolVersion)) { - long ts_ns = Micros.floor("2025-11-19T10:55:24.123456000Z") * 1000; - long ts_us = Micros.floor("2025-11-19T10:55:24.123456000Z"); - long ts_ms = Micros.floor("2025-11-19T10:55:24.123Z") / 1000; - Instant ts_instant = Instant.ofEpochSecond(ts_ns / 1_000_000_000, ts_ns % 1_000_000_000 + 10); - - if (unit != null) { - sender.table("test_tab1") - .doubleColumn("col1", 1.111) - .timestampColumn("ts_ns", ts_ns, ChronoUnit.NANOS) - .timestampColumn("ts_us", ts_us, ChronoUnit.MICROS) - .timestampColumn("ts_ms", ts_ms, ChronoUnit.MILLIS) - .timestampColumn("ts_instant", ts_instant) - .at(timestamp, unit); - } else { - sender.table("test_tab1") - .doubleColumn("col1", 1.111) - .timestampColumn("ts_ns", ts_ns, ChronoUnit.NANOS) - .timestampColumn("ts_us", ts_us, ChronoUnit.MICROS) - .timestampColumn("ts_ms", ts_ms, ChronoUnit.MILLIS) - .timestampColumn("ts_instant", ts_instant) - .at(Instant.ofEpochSecond(timestamp / 1_000_000_000, timestamp % 1_000_000_000)); - } - - sender.flush(); - } - - assertTableSizeEventually("test_tab1", 1); - assertSqlEventually("column\ttype\n" + - "col1\tDOUBLE\n" + - "timestamp\t" + ColumnType.nameOf(expectedColumnTypes[2]) + "\n" + - "ts_instant\t" + ColumnType.nameOf(expectedColumnTypes[1]) + "\n" + - "ts_ms\tTIMESTAMP\n" + - "ts_ns\t" + ColumnType.nameOf(expectedColumnTypes[0]) + "\n" + - "ts_us\tTIMESTAMP\n", - "select \"column\", \"type\" from table_columns('test_tab1') order by \"column\""); - assertSqlEventually("col1\tts_ns\tts_us\tts_ms\tts_instant\ttimestamp\n" + expected + "\n", - "test_tab1"); - } - - private void testTimestampIngest(String tableName, String timestampType, int protocolVersion, String expected1, - String expected2) throws Exception { - useTable(tableName); - execute("create table " + tableName + " (ts " + timestampType + ", dts " + timestampType - + ") timestamp(dts) partition by DAY BYPASS WAL"); - assertTableExistsEventually(tableName); - - try (Sender sender = createTcpSender(protocolVersion)) { - long ts_ns = Micros.floor("2025-11-19T10:55:24.123456000Z") * 1000; - long dts_ns = Micros.floor("2025-11-20T10:55:24.834129000Z") * 1000; - long ts_us = Micros.floor("2025-11-19T10:55:24.123456000Z"); - long dts_us = Micros.floor("2025-11-20T10:55:24.834129000Z"); - long ts_ms = Micros.floor("2025-11-19T10:55:24.123Z") / 1000; - long dts_ms = Micros.floor("2025-11-20T10:55:24.834Z") / 1000; - Instant tsInstant_ns = Instant.ofEpochSecond(ts_ns / 1_000_000_000, ts_ns % 1_000_000_000 + 10); - Instant dtsInstant_ns = Instant.ofEpochSecond(dts_ns / 1_000_000_000, dts_ns % 1_000_000_000 + 10); - - sender.table(tableName) - .timestampColumn("ts", ts_ns, ChronoUnit.NANOS) - .at(dts_ns, ChronoUnit.NANOS); - sender.table(tableName) - .timestampColumn("ts", ts_us, ChronoUnit.MICROS) - .at(dts_ns, ChronoUnit.NANOS); - sender.table(tableName) - .timestampColumn("ts", ts_ms, ChronoUnit.MILLIS) - .at(dts_ns, ChronoUnit.NANOS); - - sender.table(tableName) - .timestampColumn("ts", ts_ns, ChronoUnit.NANOS) - .at(dts_us, ChronoUnit.MICROS); - sender.table(tableName) - .timestampColumn("ts", ts_us, ChronoUnit.MICROS) - .at(dts_us, ChronoUnit.MICROS); - sender.table(tableName) - .timestampColumn("ts", ts_ms, ChronoUnit.MILLIS) - .at(dts_us, ChronoUnit.MICROS); - - sender.table(tableName) - .timestampColumn("ts", ts_ns, ChronoUnit.NANOS) - .at(dts_ms, ChronoUnit.MILLIS); - sender.table(tableName) - .timestampColumn("ts", ts_us, ChronoUnit.MICROS) - .at(dts_ms, ChronoUnit.MILLIS); - sender.table(tableName) - .timestampColumn("ts", ts_ms, ChronoUnit.MILLIS) - .at(dts_ms, ChronoUnit.MILLIS); - - sender.table(tableName) - .timestampColumn("ts", tsInstant_ns) - .at(dtsInstant_ns); - - sender.flush(); - - assertTableSizeEventually(tableName, 10); - assertSqlEventually(expected1, "select ts, dts from " + tableName); - - try { - // fails for nanos, long overflow - long ts_tooLargeForNanos_us = Micros.floor("2300-11-19T10:55:24.123456000Z"); - long dts_tooLargeForNanos_us = Micros.floor("2300-11-20T10:55:24.834129000Z"); - sender.table(tableName) - .timestampColumn("ts", ts_tooLargeForNanos_us, ChronoUnit.MICROS) - .at(dts_tooLargeForNanos_us, ChronoUnit.MICROS); - sender.flush(); - - if (expected2 == null && protocolVersion == PROTOCOL_VERSION_V1) { - Assert.fail("Exception expected"); - } - } catch (ArithmeticException e) { - if (expected2 == null && protocolVersion == PROTOCOL_VERSION_V1) { - TestUtils.assertContains(e.getMessage(), "long overflow"); - } else { - throw e; - } - } - - assertTableSizeEventually(tableName, expected2 == null ? 10 : 11); - assertSqlEventually(expected2 == null ? expected1 : expected2, "select ts, dts from " + tableName); - } - } - - private void testValueCannotBeInsertedToUuidColumn(String tableName, String value) throws Exception { - useTable(tableName); - execute("CREATE TABLE " + tableName + " (" + - "u1 UUID," + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY NONE BYPASS WAL"); - - assertTableExistsEventually(tableName); - - // this sender fails as the string is not UUID - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table(tableName) - .stringColumn("u1", value) - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - // this sender succeeds as the string is in the UUID format - try (Sender sender = createTcpSender(PROTOCOL_VERSION_V2)) { - long tsMicros = Micros.floor("2022-02-25"); - sender.table(tableName) - .stringColumn("u1", "11111111-1111-1111-1111-111111111111") - .at(tsMicros, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(tableName, 1); - assertSqlEventually( - "u1\tts\n" + - "11111111-1111-1111-1111-111111111111\t2022-02-25T00:00:00.000000000Z\n", - "select u1, ts from " + tableName); - } - private static class DummyLineChannel implements LineChannel { private int closeCounter; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/AbstractLineUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/AbstractLineUdpSenderTest.java deleted file mode 100644 index 006818a..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/AbstractLineUdpSenderTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.udp; - -import io.questdb.client.cutlass.line.AbstractLineSender; -import io.questdb.client.cutlass.line.LineUdpSender; -import io.questdb.client.std.Numbers; -import io.questdb.client.std.NumericException; -import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; - -/** - * Base class for UDP sender integration tests. - * Provides helper methods for creating UDP senders and managing test tables. - *

      - * Note: UDP is a fire-and-forget protocol, so tests need extra delays - * to account for network latency and server processing time. - */ -public abstract class AbstractLineUdpSenderTest extends AbstractLineSenderTest { - - // Default buffer capacity for UDP sender - protected static final int DEFAULT_BUFFER_CAPACITY = 2048; - - // Default TTL for multicast - protected static final int DEFAULT_TTL = 1; - - /** - * Get localhost IPv4 address as integer. - */ - protected static int getLocalhostIPv4() { - return parseIPv4("127.0.0.1"); - } - - /** - * Parse IPv4 address to integer representation. - */ - protected static int parseIPv4(String address) { - try { - return Numbers.parseIPv4(address); - } catch (NumericException e) { - throw new IllegalArgumentException("Invalid IPv4 address: " + address, e); - } - } - - /** - * Create a UDP sender for multicast. - * - * @param interfaceAddress the interface address to bind to - * @param multicastAddress the multicast group address - * @param bufferCapacity the buffer capacity in bytes - * @param ttl time-to-live for multicast packets - */ - protected AbstractLineSender createMulticastUdpSender( - int interfaceAddress, - int multicastAddress, - int bufferCapacity, - int ttl - ) { - return new LineUdpSender( - interfaceAddress, - multicastAddress, - getIlpUdpPort(), - bufferCapacity, - ttl - ); - } - - /** - * Create a UDP sender with specified buffer size. - * - * @param bufferCapacity the buffer capacity in bytes - */ - protected AbstractLineSender createUdpSender(int bufferCapacity) { - return new LineUdpSender( - getLocalhostIPv4(), // interface address - getTargetIPv4(), // target address - getIlpUdpPort(), // target port - bufferCapacity, - DEFAULT_TTL - ); - } - - /** - * Create a UDP sender with default settings. - * Uses localhost as both interface and target address. - */ - protected AbstractLineSender createUdpSender() { - return createUdpSender(DEFAULT_BUFFER_CAPACITY); - } - - /** - * Get target IPv4 address for UDP sender. - */ - protected int getTargetIPv4() { - return parseIPv4(getQuestDbHost()); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/LineUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/udp/LineUdpSenderTest.java deleted file mode 100644 index 246aa36..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/udp/LineUdpSenderTest.java +++ /dev/null @@ -1,212 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.line.udp; - -import io.questdb.client.cutlass.line.AbstractLineSender; -import org.junit.Test; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -/** - * Integration tests for UDP line sender. - *

      - * Note: UDP is a fire-and-forget protocol, so tests need extra delays - * to account for network latency and server processing time. - * These tests require an external QuestDB instance. - */ -public class LineUdpSenderTest extends AbstractLineUdpSenderTest { - - @Test - public void testAllColumnTypes() throws Exception { - String tableName = "test_udp_types"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("sym", "abc") - .longColumn("long_col", 42) - .doubleColumn("double_col", 3.14) - .stringColumn("string_col", "hello") - .boolColumn("bool_col", true) - .timestampColumn("ts_col", 1234567890L, ChronoUnit.MICROS) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testCloseAndAssertHelper() throws Exception { - String tableName = "test_udp_close"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("device", "dev1") - .longColumn("reading", 100) - .atNow(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testExplicitTimestamp() throws Exception { - String tableName = "test_udp_ts"; - useTable(tableName); - long ts = Instant.now().toEpochMilli() * 1000; - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("city", "paris") - .longColumn("temp", 15) - .at(ts, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testFlushAndAssertHelper() throws Exception { - String tableName = "udp_helper"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("sensor", "s1") - .doubleColumn("value", 123.456) - .atNow(); - flushAndAssertRowCount(sender, tableName, 1); - - sender.table(tableName) - .symbol("sensor", "s2") - .doubleColumn("value", 789.012) - .atNow(); - flushAndAssertRowCount(sender, tableName, 2); - } - } - - @Test - public void testInstantTimestamp() throws Exception { - String tableName = "udp_instant"; - useTable(tableName); - Instant now = Instant.now(); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("city", "berlin") - .longColumn("temp", 20) - .at(now); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testMultipleFlushes() throws Exception { - String tableName = "udp_multiflush"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - for (int batch = 0; batch < 5; batch++) { - for (int i = 0; i < 10; i++) { - sender.table(tableName) - .symbol("batch", String.valueOf(batch)) - .longColumn("idx", i) - .atNow(); - } - sender.flush();// Wait between batches - } - } - assertTableSizeEventually(tableName, 50); - } - - @Test - public void testMultipleRows() throws Exception { - String tableName = "udp_multi"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - for (int i = 0; i < 10; i++) { - sender.table(tableName) - .symbol("city", "city_" + i) - .longColumn("temp", i * 10) - .atNow(); - } - sender.flush(); - } - assertTableSizeEventually(tableName, 10); - } - - @Test - public void testNullStringValue() throws Exception { - String tableName = "udp_nullstr"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("id", "1") - .stringColumn("data", null) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testSimpleInsert() throws Exception { - String tableName = "udp_simple"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("city", "london") - .longColumn("temp", 42) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testSpecialCharactersInSymbol() throws Exception { - String tableName = "udp_special"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("name", "hello world") - .symbol("path", "/path/to/file") - .longColumn("count", 1) - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } - - @Test - public void testUnicodeInString() throws Exception { - String tableName = "udp_unicode"; - useTable(tableName); - try (AbstractLineSender sender = createUdpSender()) { - sender.table(tableName) - .symbol("lang", "ja") - .stringColumn("text", "こんにちは世界") - .atNow(); - sender.flush(); - } - assertTableSizeEventually(tableName, 1); - } -} From 58df95f5fafeff74bdd6f0d6c8ec02df6fc99ddb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 12 Mar 2026 15:51:01 +0100 Subject: [PATCH 187/230] Remove external server setup from CI and tests The tests that required an externally running QuestDB instance have been migrated to the core QuestDB module, where they run against an embedded server. The infrastructure that supported the old approach is no longer needed here. Deleted files: - AbstractQdbTest and AbstractLineSenderTest: dead test base classes with no remaining subclasses or callers. - ci/questdb_start.yaml, ci/questdb_stop.yaml: scripts that started and stopped a QuestDB process around each test run. - ci/confs/: default and authenticated server.conf files (and the authDb.txt credential file) used to configure those server instances. Updated files: - ci/run_client_tests.yaml: removed the configPath parameter and the start/stop template calls; the step now just runs the Maven tests. - ci/run_tests_pipeline.yaml: removed QUESTDB_RUNNING and QUESTDB_ILP_TCP_AUTH_ENABLE variables, collapsed the two run_client_tests.yaml invocations (default + authenticated) into one, and removed the "Enable ILP TCP Auth" step between them. Co-Authored-By: Claude Opus 4.6 --- ci/confs/authenticated/authDb.txt | 18 - ci/confs/authenticated/server.conf | 1396 ----------------- ci/confs/default/server.conf | 1395 ---------------- ci/questdb_start.yaml | 76 - ci/questdb_stop.yaml | 39 - ci/run_client_tests.yaml | 8 - ci/run_tests_pipeline.yaml | 10 - .../questdb/client/test/AbstractQdbTest.java | 674 -------- .../cutlass/line/AbstractLineSenderTest.java | 127 -- 9 files changed, 3743 deletions(-) delete mode 100644 ci/confs/authenticated/authDb.txt delete mode 100644 ci/confs/authenticated/server.conf delete mode 100644 ci/confs/default/server.conf delete mode 100644 ci/questdb_start.yaml delete mode 100644 ci/questdb_stop.yaml delete mode 100644 core/src/test/java/io/questdb/client/test/AbstractQdbTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java diff --git a/ci/confs/authenticated/authDb.txt b/ci/confs/authenticated/authDb.txt deleted file mode 100644 index 2e33102..0000000 --- a/ci/confs/authenticated/authDb.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Test auth db file, format is -# [key/user id] [key type] {key details} ... -# Only elliptic curve (for curve P-256) are supported (key type ec-p-256-sha256), the key details for such a key are the base64url encoded x and y points that determine the public key as defined in the JSON web token standard (RFC 7519) -# -# The auth db file needs to be put somewhere in the questdbn server root and referenced in the line.tcp.auth.db.path setting of server conf, like: -# line.tcp.auth.db.path=conf/authDb.txt -# -# Below is an elliptic curve (for curve P-256) JSON Web Key -#{ -# "kty": "EC", -# "d": "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", -# "crv": "P-256", -# "kid": "testUser1", -# "x": "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", -# "y": "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac" -#} -# For this kind of key the "d" parameter is used to generate the secret key. The "x" and "y" parameters are used to generate the public key -testUser1 ec-p-256-sha256 AKfkxOBlqBN8uDfTxu2Oo6iNsOPBnXkEH4gt44tBJKCY AL7WVjoH-IfeX_CXo5G1xXKp_PqHUrdo3xeRyDuWNbBX diff --git a/ci/confs/authenticated/server.conf b/ci/confs/authenticated/server.conf deleted file mode 100644 index a1a19b1..0000000 --- a/ci/confs/authenticated/server.conf +++ /dev/null @@ -1,1396 +0,0 @@ -# Comment or set to false to allow QuestDB to start even in the presence of config errors. -config.validation.strict=true - -# toggle whether worker should stop on error -#shared.worker.haltOnError=false - -# Number of threads in Network shared thread pool. Network thread pool used for handling HTTP, TCP, UDP and Postgres connections unless dedicated thread pools are configured. -#shared.network.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.network.worker.count" used for Network thread pool. By default, threads have no CPU affinity -#shared.network.worker.affinity= - -# Number of threads in Query shared thread pool. Query thread pool used for handling parallel queries, like parallel filters and group by queries. -#shared.query.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.query.worker.count" used for Query thread pool. By default, threads have no CPU affinity -#shared.query.worker.affinity= - -# Number of threads in Write shared thread pool. Write pool threads are used for running WAL Apply work load to merge data from WAL files into the table -#shared.write.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.write.worker.count" used for Write thread pool. By default, threads have no CPU affinity -#shared.write.worker.affinity= - -# Default number of worker threads in Network, Query and Write shared pools. Single value to configure all three pools sizes. -#shared.worker.count=2 - -# RAM usage limit, as a percentage of total system RAM. A zero value does not -# set any limit. The default is 90. -#ram.usage.limit.percent=90 - -# RAM usage limit, in bytes. A zero value (the default) does not set any limit. -# If both this and ram.usage.limit.percent are non-zero, the lower limit takes precedence. -#ram.usage.limit.bytes=0 - -# Repeats compatible migrations from the specified version. The default setting of 426 allows to upgrade and downgrade QuestDB in the range of versions from 6.2.0 to 7.0.2. -# If set to -1 start time improves but downgrades to versions below 7.0.2 and subsequent upgrades can lead to data corruption and crashes. -#cairo.repeat.migration.from.version=426 - -################ HTTP settings ################## - -# enable HTTP server -http.enabled=true - -# IP address and port of HTTP server -#http.net.bind.to=0.0.0.0:9000 - -# Uncomment to enable HTTP Basic authentication -#http.user=admin -#http.password=quest - -# Maximum time interval for the HTTP server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.net.accept.loop.timeout=500 - -#http.net.connection.limit=256 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#http.net.connection.hint=false - - -# Maximum HTTP connections that can be used for ILP ingestion using /write http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.ilp.connection.limit=-1 - -# Maximum HTTP connections that can be used for queries using /query http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.json.query.connection.limit=-1 - -# Maximum HTTP connections that can be used for export using /exp http endpoint. This limit must be lower or equal to http.net.connection.limit -# Restricted to number or CPUs or 25% of overall http connections, whichever is lower, by default. Database restart is NOT required when this setting is changed -#http.export.connection.limit=-1 - -# Idle HTTP connection timeout in milliseconds. -#http.net.connection.timeout=5m - -#Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#http.net.connection.queue.timeout=5000 - -# SO_SNDBUF value, -1 = OS default -#http.net.connection.sndbuf=2m - -# SO_RCVBUF value, -1 = OS default -#http.net.connection.rcvbuf=2m - -# size of receive buffer on application side -#http.receive.buffer.size=1m - -# initial size of the connection pool -#http.connection.pool.initial.capacity=4 - -# initial size of the string pool shared by HttpHeaderParser and HttpMultipartContentParser -#http.connection.string.pool.capacity=128 - -# HeaderParser buffer size in bytes -#http.multipart.header.buffer.size=512 - -# how long code accumulates incoming data chunks for column and delimiter analysis -#http.multipart.idle.spin.count=10000 - -#http.request.header.buffer.size=64k - -#http.worker.count=0 -#http.worker.affinity= -#http.worker.haltOnError=false - -# size of send data buffer -#http.send.buffer.size=2m - -# sets the clock to always return zero -#http.frozen.clock=false - -#http.allow.deflate.before.send=false - -# HTTP session timeout -#http.session.timeout=30m - -## When you using SSH tunnel you might want to configure -## QuestDB HTTP server to switch to HTTP/1.0 - -## Set HTTP protocol version to HTTP/1.0 -#http.version=HTTP/1.1 -## Set server keep alive to 'false'. This will make server disconnect client after -## completion of each request -#http.server.keep.alive=true - -## When in HTTP/1.0 mode keep alive values must be 0 -#http.keep-alive.timeout=5 -#http.keep-alive.max=10000 - -#http.static.public.directory=public - -#http.text.date.adapter.pool.capacity=16 -#http.text.json.cache.limit=16384 -#http.text.json.cache.size=8192 -#http.text.max.required.delimiter.stddev=0.1222d -#http.text.max.required.line.length.stddev=0.8 -#http.text.metadata.string.pool.capacity=128 -#http.text.roll.buffer.limit=8216576 -#http.text.roll.buffer.size=1024 -#http.text.analysis.max.lines=1000 -#http.text.lexer.string.pool.capacity=64 -#http.text.timestamp.adapter.pool.capacity=64 -#http.text.utf8.sink.size=4096 - -#http.json.query.connection.check.frequency=1000000 - -# enables the query cache -#http.query.cache.enabled=true - -# sets the number of blocks for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.block.count= 8 * worker_count - -# sets the number of rows for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.row.count= 2 * worker_count - -# sets the /settings endpoint readonly -#http.settings.readonly=false - -#http.security.readonly=false -#http.security.max.response.rows=Long.MAX_VALUE - -# Context path for the Web Console. If other REST services remain on the -# default context paths they will move to the same context path as the Web Console. -# Exception is ILP HTTP services, which are not used by the Web Console. They will -# remain on their default context paths. When the default context paths are changed, -# moving the Web Console will not affect the configured paths. QuestDB will create -# a copy of those services on the paths used by the Web Console so the outcome is -# both the Web Console and the custom service are operational. -#http.context.web.console=/ - -# Context path of the file import service -#http.context.import=/imp -# This service is used by the import UI in the Web Console -#http.context.table.status=/chk - -# Context path of the SQL result CSV export service -#http.context.export=/exp - -# This service provides server-side settings to the Web Console -#http.context.settings=/settings - -# SQL execution service -#http.context.execute=/exec - -# Web Console specific service -#http.context.warnings=/warnings - -# ILP HTTP Services. These are not used by the Web Console -#http.context.ilp=/write,/api/v2/write -#http.context.ilp.ping=/ping - -# Custom HTTP redirect service. All redirects are 301 - Moved permanently -#http.redirect.count=1 -#http.redirect.1=/ -> /index.html - -# circuit breaker is a mechanism that interrupts query execution -# at present queries are interrupted when remote client disconnects or when execution takes too long -# and times out - -# circuit breaker is designed to be invoke continuously in a tight loop -# the throttle is a number of pin cycles before abort conditions are tested -#circuit.breaker.throttle=2000000 - -# buffer used by I/O dispatchers and circuit breakers to check the socket state, please do not change this value -# the check reads \r\n from the input stream and discards it since some HTTP clients send this as a keep alive in between requests -#net.test.connection.buffer.size=64 - -# max execution time for read-only query in seconds -# "insert" type of queries are not aborted unless they -# it is "insert as select", where select takes long time before producing rows for the insert -query.timeout=5s - -## HTTP MIN settings -## -## Use this port to health check QuestDB instance when it isn't desired to log these health check requests. This is sort of /dev/null for monitoring - -#http.min.enabled=true -#http.min.net.bind.to=0.0.0.0:9003 - -# When enabled, health check will return HTTP 500 if there were any unhandled errors since QuestDB instance start. -#http.pessimistic.health.check.enabled=false - -# Maximum time interval for the HTTP MIN server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.min.net.accept.loop.timeout=500 - -################ Cairo settings ################## - -# directory for storing db tables and metadata. this directory is inside the server root directory provided at startup -#cairo.root=db - -# how changes to table are flushed to disk upon commit - default: nosync. Choices: nosync, async (flush call schedules update, returns immediately), sync (waits for flush to complete) -#cairo.commit.mode=nosync - -# The amount of time server is allowed to be compounding transaction before physical commit is forced. -# Compounding of the transactions improves system's through but hurts latency. Reduce this value to -# reduce latency of data visibility. -#cairo.commit.latency=30s - -# number of types table creation or insertion will be attempted -#cairo.create.as.select.retry.count=5 - -# comma separated list of volume definitions, volume_alias -> absolute_path_to_existing_directory. -# volume alias can then be used in create table statement with IN VOLUME clause -#cairo.volumes= by default IN VOLUME is switched off, no volume definitions. - -# type of map uses. Options: 1. fast (speed at the expense of storage. this is the default option) 2. compact -#cairo.default.map.type=fast - -# when true, symbol values will be cached on Java heap -#cairo.default.symbol.cache.flag=false - -# when column type is SYMBOL this parameter specifies approximate capacity for symbol map. -# It should be equal to number of unique symbol values stored in the table and getting this -# value badly wrong will cause performance degradation. Must be power of 2 -#cairo.default.symbol.capacity=256 - -# number of attempts to open files -#cairo.file.operation.retry.count=30 - -# when DB is running in sync/async mode, how many 'steps' IDGenerators will pre-allocate and synchronize sync to disk -#cairo.id.generator.batch.step=512 - -# how often the writer maintenance job gets run -#cairo.idle.check.interval=5m - -# defines the number of latest partitions to keep open when returning a reader to the reader pool -#cairo.inactive.reader.max.open.partitions=128 - -# defines frequency in milliseconds with which the reader pool checks for inactive readers. -#cairo.inactive.reader.ttl=2m - -# defines frequency in milliseconds with which the writer pool checks for inactive readers. -#cairo.inactive.writer.ttl=10m - -# when true (default), TTL enforcement uses wall clock time to prevent accidental data loss -# when future timestamps are inserted. When false, TTL uses only the max timestamp in the table. -#cairo.ttl.use.wall.clock=true - -# approximation of number of rows for single index key, must be power of 2 -#cairo.index.value.block.size=256 - -# number of attempts to open swap file -#cairo.max.swap.file.count=30 - -# file permission for new directories -#cairo.mkdir.mode=509 - -# Time to wait before retrying writing into a table after a memory limit failure -#cairo.write.back.off.timeout.on.mem.pressure=4s - -# maximum file name length in chars. Affects maximum table name length and maximum column name length -#cairo.max.file.name.length=127 - -# minimum number of rows before allowing use of parallel indexation -#cairo.parallel.index.threshold=100000 - -# number of segments in the TableReader pool; each segment holds up to 32 readers -#cairo.reader.pool.max.segments=10 - -# timeout in milliseconds when attempting to get atomic memory snapshots, e.g. in BitmapIndexReaders -#cairo.spin.lock.timeout=1s - -# sets size of the CharacterStore -#cairo.character.store.capacity=1024 - -# Sets size of the CharacterSequence pool -#cairo.character.store.sequence.pool.capacity=64 - -# sets size of the Column pool in the SqlCompiler -#cairo.column.pool.capacity=4096 - -# size of the ExpressionNode pool in SqlCompiler -#cairo.expression.pool.capacity=8192 - -# load factor for all FastMaps -#cairo.fast.map.load.factor=0.7 - -# size of the JoinContext pool in SqlCompiler -#cairo.sql.join.context.pool.capacity=64 - -# size of FloatingSequence pool in GenericLexer -#cairo.lexer.pool.capacity=2048 - -# sets the key capacity in FastMap and CompactMap -#cairo.sql.map.key.capacity=2097152 - -# sets the key capacity in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.key.capacity=32 - -# number of map resizes in FastMap and CompactMap before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.map.max.resizes=2^31 - -# memory page size for FastMap and CompactMap -#cairo.sql.map.page.size=4m - -# memory page size in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.page.size=32k - -# memory max pages for CompactMap -#cairo.sql.map.max.pages=2^31 - -# sets the size of the QueryModel pool in the SqlCompiler -#cairo.model.pool.capacity=1024 - -# sets the maximum allowed negative value used in LIMIT clause in queries with filters -#cairo.sql.max.negative.limit=10000 - -# sets the memory page size for storing keys in LongTreeChain -#cairo.sql.sort.key.page.size=128k - -# max number of pages for storing keys in LongTreeChain before a resource limit exception is thrown -# cairo.sql.sort.key.max.pages=2^31 - -# sets the memory page size and max pages for storing values in LongTreeChain -#cairo.sql.sort.light.value.page.size=128k -#cairo.sql.sort.light.value.max.pages=2^31 - -# sets the memory page size and max pages of the slave chain in full hash joins -#cairo.sql.hash.join.value.page.size=16777216 -#cairo.sql.hash.join.value.max.pages=2^31 - -# sets the initial capacity for row id list used for latest by -#cairo.sql.latest.by.row.count=1000 - -# sets the memory page size and max pages of the slave chain in light hash joins -#cairo.sql.hash.join.light.value.page.size=128k -#cairo.sql.hash.join.light.value.max.pages=2^31 - -# number of rows to scan linearly before starting binary search in ASOF JOIN queries with no additional keys -#cairo.sql.asof.join.lookahead=10 - -# sets memory page size and max pages of file storing values in SortedRecordCursorFactory -#cairo.sql.sort.value.page.size=16777216 -#cairo.sql.sort.value.max.pages=2^31 - -# latch await timeout in nanoseconds for stealing indexing work from other threads -#cairo.work.steal.timeout.nanos=10000 - -# whether parallel indexation is allowed. Works in conjunction with cairo.parallel.index.threshold -#cairo.parallel.indexing.enabled=true - -# memory page size for JoinMetadata file -#cairo.sql.join.metadata.page.size=16384 - -# number of map resizes in JoinMetadata before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.join.metadata.max.resizes=2^31 - -# size of PivotColumn pool in SqlParser -#cairo.sql.pivot.column.pool.capacity=64 - -# maximum number of columns PIVOT can produce (FOR value combinations × aggregates) -#cairo.sql.pivot.max.produced.columns=5000 - -# size of WindowColumn pool in SqlParser -#cairo.sql.window.column.pool.capacity=64 - -# sets the memory page size and max number of pages for records in window function -#cairo.sql.window.store.page.size=1m -#cairo.sql.window.store.max.pages=2^31 - -# sets the memory page size and max number of pages for row ids in window function -#cairo.sql.window.rowid.page.size=512k -#cairo.sql.window.rowid.max.pages=2^31 - -# sets the memory page size and max number of pages for keys in window function -#cairo.sql.window.tree.page.size=512k -#cairo.sql.window.tree.max.pages=2^31 - -# sets initial size of per-partition window function range frame buffer -#cairo.sql.window.initial.range.buffer.size=32 - -# batch size of non-atomic inserts for CREATE TABLE AS SELECT statements -#cairo.sql.create.table.model.batch.size=1000000 - -# Size of the pool for model objects, that underpin the "create table" parser. -# It is aimed at reducing allocations and it a performance setting. The -# number is aligned to the max concurrent "create table" requests the system -# will ever receive. If system receives more requests that this, it will just -# allocate more object and free them after use. -#cairo.create.table.column.model.pool.capacity=16 - -# size of RenameTableModel pool in SqlParser -#cairo.sql.rename.table.model.pool.capacity=16 - -# size of WithClauseModel pool in SqlParser -#cairo.sql.with.clause.model.pool.capacity=128 - -# size of CompileModel pool in SqlParser -#cairo.sql.compile.view.model.pool.capacity=8 - -# initial size of view lexer pool in SqlParser, used to parse SELECT statements of view definitions -# the max number of views used in a single query determines how many view lexers should be in the pool -#cairo.sql.view.lexer.pool.capacity=8 - -# size of InsertModel pool in SqlParser -#cairo.sql.insert.model.pool.capacity=64 - -# batch size of non-atomic inserts for INSERT INTO SELECT statements -#cairo.sql.insert.model.batch.size=1000000 - -# enables parallel GROUP BY execution; by default, parallel GROUP BY requires at least 4 shared worker threads to take place -#cairo.sql.parallel.groupby.enabled=true - -# merge queue capacity for parallel GROUP BY; used for parallel tasks that merge shard hash tables -#cairo.sql.parallel.groupby.merge.shard.queue.capacity= - -# threshold for parallel GROUP BY to shard the hash table holding the aggregates -#cairo.sql.parallel.groupby.sharding.threshold=10000 - -# enables statistics-based hash table pre-sizing in parallel GROUP BY -#cairo.sql.parallel.groupby.presize.enabled=true - -# maximum allowed hash table size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.capacity=100000000 - -# maximum allowed heap size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.heap.size=1G - -# threshold for parallel ORDER BY + LIMIT execution on sharded GROUP BY hash table -#cairo.sql.parallel.groupby.topk.threshold=5000000 - -# queue capacity for parallel ORDER BY + LIMIT applied to sharded GROUP BY -#cairo.sql.parallel.groupby.topk.queue.capacity= - -# threshold for in-flight tasks for disabling work stealing during parallel SQL execution -# when the number of shared workers is less than 4x of this setting, work stealing is always enabled -#cairo.sql.parallel.work.stealing.threshold=16 - -# spin timeout in nanoseconds for adaptive work stealing strategy -# controls how long the query thread waits for worker threads to pick up tasks before stealing work back -#cairo.sql.parallel.work.stealing.spin.timeout=50000 - -# enables parallel read_parquet() SQL function execution; by default, parallel read_parquet() requires at least 4 shared worker threads to take place -#cairo.sql.parallel.read.parquet.enabled=true - -# capacity for Parquet page frame cache; larger values may lead to better ORDER BY and some other -# clauses performance at the cost of memory overhead -#cairo.sql.parquet.frame.cache.capacity=3 - -# default size for memory buffers in GROUP BY function native memory allocator -#cairo.sql.groupby.allocator.default.chunk.size=128K - -# maximum allowed native memory allocation for GROUP BY functions -#cairo.sql.groupby.allocator.max.chunk.size=4G - -# threshold in bytes for switching from single memory buffer hash table (unordered) to a hash table with separate heap for entries (ordered) -#cairo.sql.unordered.map.max.entry.size=32 - -## prevents stack overflow errors when evaluating complex nested SQLs -## the value is an approximate number of nested SELECT clauses. -#cairo.sql.window.max.recursion=128 - -## pre-sizes the internal data structure that stores active query executions -## the value is chosen automatically based on the number of threads in the shared worker pool -#cairo.sql.query.registry.pool.size= - -## window function buffer size in record counts -## pre-sizes buffer for every windows function execution to contain window records -#cairo.sql.analytic.initial.range.buffer.size=32 - -## enables quick and radix sort in order by, when applicable -#cairo.sql.orderby.sort.enabled=true - -## defines number of rows to use radix sort in order by -#cairo.sql.orderby.radix.sort.threshold=600 - -## enables the column alias to be generated from the expression -#cairo.sql.column.alias.expression.enabled=true - -## maximum length of generated column aliases -#cairo.sql.column.alias.generated.max.size=64 - -## initial capacity of string pool for preferences store and parser -#cairo.preferences.string.pool.capacity=64 - -## Flag to enable or disable symbol capacity auto-scaling. Auto-scaling means resizing -## symbol table data structures as the number of symbols in the table grows. Optimal sizing of -## these data structures ensures optimal ingres performance. -## -## By default, the auto-scaling is enabled. This is optimal. You may want to disable auto-scaling in case -## something goes wrong. -## -## Database restart is NOT required when this setting is changed, but `reload_config()` SQL should be executed. -#cairo.auto.scale.symbol.capacity=true - -## Symbol occupancy threshold after which symbol capacity is doubled. For example -## threshold of 0.8 means that occupancy have to reach 80% of capacity before capacity is increased -#cairo.auto.scale.symbol.capacity.threshold=0.8 - -#### SQL COPY - -# size of CopyModel pool in SqlParser -#cairo.sql.copy.model.pool.capacity=32 - -# size of buffer used when copying tables -#cairo.sql.copy.buffer.size=2m - -# name of file with user's set of date and timestamp formats -#cairo.sql.copy.formats.file=/text_loader.json - -# input root directory, where COPY command and read_parquet() function read files from -# relative paths are resolved against the server root directory -cairo.sql.copy.root=import - -# export root directory, where COPY .. to command write files to -# relative paths are resolved against the server root directory -#cairo.sql.copy.export.root=export - -# input work directory, where temporary import files are created, by default it's located in tmp directory inside the server root directory -#cairo.sql.copy.work.root=null - -# default max size of intermediate import file index chunk (100MB). Import shouldn't use more memory than worker_count * this . -#cairo.sql.copy.max.index.chunk.size=100M - -# Capacity of the internal queue used to split parallel copy SQL command into subtasks and execute them across shared worker threads. -# The default configuration should be suitable for importing files of any size. -#cairo.sql.copy.queue.capacity=32 - -# Capacity of the internal queue used to execute copy export SQL command. -#cairo.sql.copy.export.queue.capacity=32 - -# Frequency of logging progress when exporting data using COPY .. TO command to export to parquet format. 0 or negative value disables the logging. -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.copy.report.frequency.lines=50000 - -# Parquet version to use when exporting data using COPY .. TO command. Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.parquet.export.version=1 - -# Enable statistics collection in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.raw.array.encoding.enabled=false - -# Compression codec to use when exporting data using COPY .. TO command. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.parquet.export.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs when exporting data using COPY .. TO command -#cairo.parquet.export.compression.level=9 - -# Row group size in rows when exporting data using COPY .. TO command. 0 uses the default (512*512 rows) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.row.group.size=0 - -# Data page size in bytes when exporting data using COPY .. TO command. 0 uses the default (1024*1024 bytes) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.data.page.size=0 - -# Maximum time to wait for export to complete when using /exp HTTP endpoint. 0 means no timeout. -#http.export.timeout=300s - -# Parquet version to use for partition encoder (for storing partitions in parquet format). Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.partition.encoder.parquet.version=1 - -# Enable statistics collection in Parquet files for partition encoder -#cairo.partition.encoder.parquet.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files for partition encoder -#cairo.partition.encoder.parquet.raw.array.encoding.enabled=false - -# Compression codec to use for partition encoder. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.partition.encoder.parquet.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs for partition encoder -#cairo.partition.encoder.parquet.compression.level=9 - -# Row group size in rows for partition encoder -#cairo.partition.encoder.parquet.row.group.size=100000 - -# Data page size in bytes for partition encoder -#cairo.partition.encoder.parquet.data.page.size=1048576 - -# Number of days to retain records in import log table (sys.parallel_text_import_log). Old records get deleted on each import and server restart. -#cairo.sql.copy.log.retention.days=3 - -# output root directory for backups -#cairo.sql.backup.root=null - -# date format for backup directory -#cairo.sql.backup.dir.datetime.format=yyyy-MM-dd - -# name of temp directory used during backup -#cairo.sql.backup.dir.tmp.name=tmp - -# permission used when creating backup directories -#cairo.sql.backup.mkdir.mode=509 - -# suffix of the partition directory in detached root to indicate it is ready to be attached -#cairo.attach.partition.suffix=.attachable - -# Use file system "copy" operation instead of "hard link" when attaching partition from detached root. Set to true if detached root is on a different drive. -#cairo.attach.partition.copy=false - -# file permission used when creating detached directories -#cairo.detached.mkdir.mode=509 - -# sample by index query page size - max values returned in single scan -# 0 means to use symbol block capacity -# cairo.sql.sampleby.page.size=0 - -# sample by default alignment -# if true, sample by will default to SAMPLE BY (FILL) ALIGN TO CALENDAR -# if false, sample by will default to SAMPLE BY (FILL) ALIGN TO FIRST OBSERVATION -# cairo.sql.sampleby.default.alignment.calendar=true - -# sets the minimum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.min.rows=100000 - -# sets the maximum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.max.rows=1000000 - -# sets the minimum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.min.rows=10000 - -# sets the maximum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.max.rows=100000 - -# sets the memory page size and max number of pages for memory used by rnd functions -# currently rnd_str() and rnd_symbol(), this could extend to other rnd functions in the future -#cairo.rnd.memory.page.size=8K -#cairo.rnd.memory.max.pages=128 - -# max length (in chars) of buffer used to store result of SQL functions, such as replace() or lpad() -#cairo.sql.string.function.buffer.max.size=1048576 - -# SQL JIT compiler mode. Options: -# 1. on (enable JIT and use vector instructions when possible; default value) -# 2. scalar (enable JIT and use scalar instructions only) -# 3. off (disable JIT) -#cairo.sql.jit.mode=on - -# sets the memory page size and max pages for storing IR for JIT compilation -#cairo.sql.jit.ir.memory.page.size=8K -#cairo.sql.jit.ir.memory.max.pages=8 - -# sets the memory page size and max pages for storing bind variable values for JIT compiled filter -#cairo.sql.jit.bind.vars.memory.page.size=4K -#cairo.sql.jit.bind.vars.memory.max.pages=8 - -# sets debug flag for JIT compilation; when enabled, assembly will be printed into stdout -#cairo.sql.jit.debug.enabled=false - -# Controls the maximum allowed IN list length before the JIT compiler will fall back to Java code -#cairo.sql.jit.max.in.list.size.threshold=10 - -#cairo.date.locale=en - -# Maximum number of uncommitted rows in TCP ILP -#cairo.max.uncommitted.rows=500000 - -# Minimum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MinLag is the lower limit -#cairo.o3.min.lag=1s - -# Maximum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MaxLag is the upper limit -#cairo.o3.max.lag=600000 - -# Memory page size per column for O3 operations. Please be aware O3 will use 2x of this RAM per column -#cairo.o3.column.memory.size=8M - -# Memory page size per column for O3 operations on System tables only -#cairo.system.o3.column.memory.size=256k - -# Number of partition expected on average, initial value for purge allocation job, extended in runtime automatically -#cairo.o3.partition.purge.list.initial.capacity=1 - -# mmap sliding page size that TableWriter uses to append data for each column -#cairo.writer.data.append.page.size=16M - -# mmap page size for mapping small files, such as _txn, _todo and _meta -# the default value is OS page size (4k Linux, 64K windows, 16k OSX M1) -# if you override this value it will be rounded to the nearest (greater) multiple of OS page size -#cairo.writer.misc.append.page.size=4k - -# mmap page size for appending index key data; key data is number of distinct symbol values times 4 bytes -#cairo.writer.data.index.key.append.page.size=512k - -# mmap page size for appending value data; value data are rowids, e.g. number of rows in partition times 8 bytes -#cairo.writer.data.index.value.append.page.size=16M - -# mmap sliding page size that TableWriter uses to append data for each column specifically for System tables -#cairo.system.writer.data.append.page.size=256k - -# File allocation page min size for symbol table files -#cairo.symbol.table.min.allocation.page.size=4k - -# File allocation page max size for symbol table files -#cairo.symbol.table.max.allocation.page.size=8M - -# Maximum wait timeout in milliseconds for ALTER TABLE SQL statement run via REST and PG Wire interfaces when statement execution is ASYNCHRONOUS -#cairo.writer.alter.busy.wait.timeout=500ms - -# Row count to check writer command queue after on busy writing (e.g. tick after X rows written) -#cairo.writer.tick.rows.count=1024 - -# Maximum writer ALTER TABLE and replication command capacity. Shared between all the tables -#cairo.writer.command.queue.capacity=32 - -# Sets flag to enable io_uring interface for certain disk I/O operations on newer Linux kernels (5.12+). -#cairo.iouring.enabled=true - -# Minimum O3 partition prefix size for which O3 partition split happens to avoid copying the large prefix -#cairo.o3.partition.split.min.size=50M - -# The number of O3 partition splits allowed for the last partitions. If the number of splits grows above this value, the splits will be squashed -#cairo.o3.last.partition.max.splits=20 - -################ Parallel SQL execution ################ - -# Sets flag to enable parallel SQL filter execution. JIT compilation takes place only when this setting is enabled. -#cairo.sql.parallel.filter.enabled=true - -# Sets the threshold for column pre-touch to be run as a part of the parallel SQL filter execution. The threshold defines ratio between the numbers of scanned and filtered rows. -#cairo.sql.parallel.filter.pretouch.threshold=0.05 - -# Sets the upper limit on the number of in-flight tasks for parallel query workers published by filter queries with LIMIT. -#cairo.sql.parallel.filter.dispatch.limit= - -# Sets flag to enable parallel ORDER BY + LIMIT SQL execution. -#cairo.sql.parallel.topk.enabled=true - -# Sets flag to enable parallel WINDOW JOIN SQL execution. -#cairo.sql.parallel.window.join.enabled=true - -# Shard reduce queue contention between SQL statements that are executed concurrently. -#cairo.page.frame.shard.count=4 - -# Reduce queue is used for data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.page.frame.reduce.queue.capacity= - -# Reduce queue is used for vectorized data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.vector.aggregate.queue.capacity= - -# Initial row ID list capacity for each slot of the "reduce" queue. Larger values reduce memory allocation rate, but increase RSS size. -#cairo.page.frame.rowid.list.capacity=256 - -# Initial column list capacity for each slot of the "reduce" queue. Used by JIT-compiled filters. -#cairo.page.frame.column.list.capacity=16 - -# Initial object pool capacity for local "reduce" tasks. These tasks are used to avoid blocking query execution when the "reduce" queue is full. -#cairo.page.frame.task.pool.capacity=4 - -################ LINE settings ###################### - -#line.default.partition.by=DAY - -# Enable / Disable automatic creation of new columns in existing tables via ILP. When set to false overrides value of line.auto.create.new.tables to false -#line.auto.create.new.columns=true - -# Enable / Disable automatic creation of new tables via ILP. -#line.auto.create.new.tables=true - -# Enable / Disable printing problematic ILP messages in case of errors. -#line.log.message.on.error=true - -################ LINE UDP settings ################## - -#line.udp.bind.to=0.0.0.0:9009 -#line.udp.join=232.1.2.3 -#line.udp.commit.rate=1000000 -#line.udp.msg.buffer.size=2048 -#line.udp.msg.count=10000 -#line.udp.receive.buffer.size=8m -line.udp.enabled=true -#line.udp.own.thread.affinity=-1 -#line.udp.own.thread=false -#line.udp.unicast=false -#line.udp.commit.mode=nosync -#line.udp.timestamp=n - -######################### LINE TCP settings ############################### - -#line.tcp.enabled=true -#line.tcp.net.bind.to=0.0.0.0:9009 -#line.tcp.net.connection.limit=256 -line.tcp.auth.db.path=conf/authDb.txt - -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#line.tcp.net.connection.hint=false - -# Idle TCP connection timeout in milliseconds. 0 means there is no timeout. -#line.tcp.net.connection.timeout=0 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#line.tcp.net.connection.queue.timeout=5000 - -# Maximum time interval for the ILP TCP endpoint to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#line.tcp.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#line.tcp.net.connection.rcvbuf=-1 - -#line.tcp.connection.pool.capacity=64 -#line.tcp.timestamp=n - -# TCP message buffer size -#line.tcp.msg.buffer.size=2048 - -# Max measurement size -#line.tcp.max.measurement.size=2048 - -# Max receive buffer size -#line.tcp.max.recv.buffer.size=1073741824 - -# Size of the queue between the IO jobs and the writer jobs, each queue entry represents a measurement -#line.tcp.writer.queue.capacity=128 - -# IO and writer job worker pool settings, 0 indicates the shared pool should be used -#line.tcp.writer.worker.count=0 -#line.tcp.writer.worker.affinity= -#line.tcp.writer.worker.yield.threshold=10 -#line.tcp.writer.worker.nap.threshold=7000 -#line.tcp.writer.worker.sleep.threshold=10000 -#line.tcp.writer.halt.on.error=false - -#line.tcp.io.worker.count=0 -#line.tcp.io.worker.affinity= -#line.tcp.io.worker.yield.threshold=10 -#line.tcp.io.worker.nap.threshold=7000 -#line.tcp.io.worker.sleep.threshold=10000 -#line.tcp.io.halt.on.error=false - -# Sets flag to disconnect TCP connection that sends malformed messages. -#line.tcp.disconnect.on.error=true - -# Commit lag fraction. Used to calculate commit interval for the table according to the following formula: -# commit_interval = commit_lag ∗ fraction -# The calculated commit interval defines how long uncommitted data will need to remain uncommitted. -#line.tcp.commit.interval.fraction=0.5 -# Default commit interval in milliseconds. Used when o3MinLag is set to 0. -#line.tcp.commit.interval.default=2000 - -# Maximum amount of time in between maintenance jobs in milliseconds, these will commit uncommitted data -#line.tcp.maintenance.job.interval=1000 -# Minimum amount of idle time before a table writer is released in milliseconds -#line.tcp.min.idle.ms.before.writer.release=500 - - -#line.tcp.symbol.cache.wait.before.reload=500ms - -######################### LINE HTTP settings ############################### - -#line.http.enabled=true - -#line.http.max.recv.buffer.size=1073741824 - -#line.http.ping.version=v2.7.4 - -################ PG Wire settings ################## - -#pg.enabled=true -#pg.net.bind.to=0.0.0.0:8812 -#pg.net.connection.limit=64 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if active.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#pg.net.connection.hint=false - -# Connection idle timeout in milliseconds. Connections are closed by the server when this timeout lapses. -#pg.net.connection.timeout=300000 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached. -#pg.net.connection.queue.timeout=5m - -# Maximum time interval for the PG Wire server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#pg.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#pg.net.connection.rcvbuf=-1 - -# SO_SNDBUF value, -1 = OS default -#pg.net.connection.sndbuf=-1 - -#pg.legacy.mode.enabled=false -#pg.character.store.capacity=4096 -#pg.character.store.pool.capacity=64 -#pg.connection.pool.capacity=64 -#pg.password=quest -#pg.user=admin -# Enables read-only mode for the pg wire protocol. In this mode data mutation queries are rejected. -#pg.security.readonly=false -#pg.readonly.password=quest -#pg.readonly.user=user -# Enables separate read-only user for the pg wire server. Data mutation queries are rejected for all connections opened by this user. -#pg.readonly.user.enabled=false -# enables select query cache -#pg.select.cache.enabled=true -# sets the number of blocks for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.block.count= 8 * worker_count -# sets the number of rows for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.row.count= 2 * worker_count -# enables insert query cache -#pg.insert.cache.enabled=true -# sets the number of blocks for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.block.count=4 -# sets the number of rows for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.row.count=4 -#pg.max.blob.size.on.query=512k -#pg.recv.buffer.size=1M -#pg.net.connection.rcvbuf=-1 -#pg.send.buffer.size=1M -#pg.net.connection.sndbuf=-1 -#pg.date.locale=en -#pg.worker.count=2 -#pg.worker.affinity=-1,-1; -#pg.halt.on.error=false -#pg.daemon.pool=true -#pg.binary.param.count.capacity=2 -# maximum number of prepared statements a single client can create at a time. This is to prevent clients from creating -# too many prepared statements and exhausting server resources. -#pg.named.statement.limit=10000 - -# if you are using insert batches of over 64 rows, you should increase this value to avoid memory resizes that -# might slow down inserts. Be careful though as this allocates objects on JavaHeap. Setting this value too large -# might kill the GC and server will be unresponsive on startup. -#pg.pipeline.capacity=64 - -################ Materialized View settings ################## - -# Enables SQL support and refresh job for materialized views. -#cairo.mat.view.enabled=true - -# When disabled, SQL executed by materialized view refresh job always runs single-threaded. -#cairo.mat.view.parallel.sql.enabled=true - -# Desired number of base table rows to be scanned by one query during materialized view refresh. -#cairo.mat.view.rows.per.query.estimate=1000000 - -# Maximum time interval used for a single step of materialized view refresh. For larger intervals single SAMPLE BY interval will be used as the step. -#cairo.mat.view.max.refresh.step=31536000000000us - -# Maximum number of times QuestDB will retry a materialized view refresh after a recoverable error. -#cairo.mat.view.max.refresh.retries=10 - -# Timeout for next refresh attempts after an out-of-memory error in a materialized view refresh, in milliseconds. -#cairo.mat.view.refresh.oom.retry.timeout=200 - -# Maximum number of rows inserted by the refresh job in a single transaction. -#cairo.mat.view.insert.as.select.batch.size=1000000 - -# Maximum number of time intervals from WAL data transactions cached per materialized view. -#cairo.mat.view.max.refresh.intervals=100 - -# Interval for periodical refresh intervals caching for materialized views. -#cairo.mat.view.refresh.intervals.update.period=15s - -# Number of parallel threads used to refresh materialized view data. -# Configuration options: -# - Default: Automatically calculated based on CPU core count -# - 0: Runs as a single instance on the shared worker pool -# - N: Uses N dedicated threads for parallel refreshing -#mat.view.refresh.worker.count= - -# Materialized View refresh worker pool configuration -#mat.view.refresh.worker.affinity= -#mat.view.refresh.worker.yield.threshold=1000 -#mat.view.refresh.worker.nap.threshold=7000 -#mat.view.refresh.worker.sleep.threshold=10000 -#mat.view.refresh.worker.haltOnError=false - -# Export worker pool configuration -#export.worker.affinity= -#export.worker.yield.threshold=1000 -#export.worker.nap.threshold=7000 -#export.worker.sleep.threshold=10000 -#export.worker.haltOnError=false - -################ WAL settings ################## - -# Enable support of creating and writing to WAL tables -#cairo.wal.supported=true - -# If set to true WAL becomes the default mode for newly created tables. Impacts table created from ILP and SQL if WAL / BYPASS WAL not specified -#cairo.wal.enabled.default=true - -# Parallel threads to apply WAL data to the table storage. By default it is equal to the CPU core count. -# When set to 0 WAL apply job will run as a single instance on shared worker pool. -#wal.apply.worker.count= - -# WAL apply pool configuration -#wal.apply.worker.affinity= -#wal.apply.worker.yield.threshold=1000 -#wal.apply.worker.nap.threshold=7000 -#wal.apply.worker.sleep.threshold=10000 -#wal.apply.worker.haltOnError=false - -# Period in ms of how often WAL applied files are cleaned up from the disk -#cairo.wal.purge.interval=30s - -# Row count of how many rows are written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.size` (whichever is first). -#cairo.wal.segment.rollover.row.count=200000 - -# Byte size of number of rows written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.row.count` (whichever is first). -# By default this is 0 (disabled) unless `replication.role=primary` is set, then it is defaulted to 2MiB. -#cairo.wal.segment.rollover.size=0 - -# mmap sliding page size that WalWriter uses to append data for each column -#cairo.wal.writer.data.append.page.size=1M - -# mmap sliding page size that WalWriter uses to append events for each column -# this page size has performance impact on large number of small transactions, larger -# page will cope better. However, if the workload is that of small number of large transaction, -# this page size can be reduced. The optimal value should be established via ingestion benchmark. -#cairo.wal.writer.event.append.page.size=128k - -# Multiplier to cairo.max.uncommitted.rows to calculate the limit of rows that can kept invisible when writing -# to WAL table under heavy load, when multiple transactions are to be applied. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.squash.uncommitted.rows.multiplier=20.0 - -# Maximum size of data can be kept in WAL lag in bytes. -# Default is 75M and it means that once the limit is reached, 50M will be committed and 25M will be kept in the lag. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.size=75M - -# Maximum number of transactions to keep in O3 lag for WAL tables. Once the number is reached, full commit occurs. -# By default it is -1 and the limit does not apply, instead the size limit of cairo.wal.max.lag.size is used. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.txn.count=-1 - -# When WAL apply job processes transactions this is the minimum number of transaction -# to look ahead and read metadata of before applying any of them. -#cairo.wal.apply.look.ahead.txn.count=20 - -# Part of WAL apply job fair factor. The amount of time job spends on single table -# before moving to the next one. -#cairo.wal.apply.table.time.quota=1s - -# number of segments in the WalWriter pool; each segment holds up to 16 writers -#cairo.wal.writer.pool.max.segments=10 - -# number of segments in the ViewWalWriter pool; each segment holds up to 16 writers -#cairo.view.wal.writer.pool.max.segments=4 - -# ViewWalWriter pool releases inactive writers after this TTL. In milliseconds. -#cairo.view.wal.inactive.writer.ttl=60000 - -# Maximum number of Segments to cache File descriptors for when applying from WAL to table storage. -# WAL segment files are cached to avoid opening and closing them on processing each commit. -# Ideally should be in line with average number of simultaneous connections writing to the tables. -#cairo.wal.max.segment.file.descriptors.cache=30 - -# When disabled, SQL executed by WAL apply job always runs single-threaded. -#cairo.wal.apply.parallel.sql.enabled=true - -################ Telemetry settings ################## - -# Telemetry events are used to identify components of QuestDB that are being -# used. They never identify user nor data stored in QuestDB. All events can be -# viewed via `select * from telemetry`. Switching telemetry off will stop all -# events from being collected. After telemetry is switched off, telemetry -# tables can be dropped. - -telemetry.enabled=false -#telemetry.queue.capacity=512 -#telemetry.hide.tables=true -#telemetry.table.ttl.weeks=4 -#telemetry.event.throttle.interval=1m - -################ Metrics settings ################## - -#metrics.enabled=false - -############# Query Tracing settings ############### - -#query.tracing.enabled=false - -################ Logging settings ################## - -# enable or disable 'exe' log messages written when query execution begins -#log.sql.query.progress.exe=true - - -################ Enterprise configuration options ################## -### Please visit https://questdb.io/enterprise/ for more information -#################################################################### - - -#acl.enabled=false -#acl.entity.name.max.length=255 -#acl.admin.user.enabled=true -#acl.admin.user=admin -#acl.admin.password=quest - -## 10 seconds -#acl.rest.token.refresh.threshold=10 -#acl.sql.permission.model.pool.capacity=32 -#acl.password.hash.iteration.count=100000 - -#acl.basic.auth.realm.enabled=false - -#cold.storage.enabled=false -#cold.storage.object.store= - -#tls.enabled=false -#tls.cert.path= -#tls.private.key.path= - -## Defaults to the global TLS settings -## including enable/disable flag and paths -## Use these configuration values to separate -## min-http server TLS configuration from global settings -#http.min.tls.enabled= -#http.min.tls.cert.path= -#http.min.tls.private.key.path= - -#http.tls.enabled=false -#http.tls.cert.path= -#http.tls.private.key.path= - -#line.tcp.tls.enabled= -#line.tcp.tls.cert.path= -#line.tcp.tls.private.key.path= - -#pg.tls.enabled= -#pg.tls.cert.path= -#pg.tls.private.key.path= - -## The number of threads dedicated for async IO operations (e.g. network activity) in native code. -#native.async.io.threads= - -## The number of threads dedicated for blocking IO operations (e.g. file access) in native code. -#native.max.blocking.threads= - -### Replication (QuestDB Enterprise Only) - -# Possible roles are "primary", "replica", or "none" -# primary - read/write node -# replica - read-only node, consuming data from PRIMARY -# none - replication is disabled -#replication.role=none - -## Object-store specific string. -## AWS S3 example: -## s3::bucket=${BUCKET_NAME};root=${DB_INSTANCE_NAME};region=${AWS_REGION};access_key_id=${AWS_ACCESS_KEY};secret_access_key=${AWS_SECRET_ACCESS_KEY}; -## Azure Blob example: -## azblob::endpoint=https://${STORE_ACCOUNT}.blob.core.windows.net;container={BLOB_CONTAINER};root=${DB_INSTANCE_NAME};account_name=${STORE_ACCOUNT};account_key=${STORE_KEY}; -## Filesystem: -## fs::root=/nfs/path/to/dir/final;atomic_write_dir=/nfs/path/to/dir/scratch; -#replication.object.store= - -## Limits the number of concurrent requests to the object store. -## Defaults to 128 -#replication.requests.max.concurrent=128 - -## Maximum number of times to retry a failed object store request before -## logging an error and reattempting later after a delay. -#replication.requests.retry.attempts=3 - -## Delay between the retry attempts (milliseconds) -#replication.requests.retry.interval=200 - -## The time window grouping multiple transactions into a replication batch (milliseconds). -## Smaller time windows use more network traffic. -## Larger time windows increase the replication latency. -## Works in conjunction with `replication.primary.throttle.non.data`. -#replication.primary.throttle.window.duration=10000 - -## Set to `false` to allow immediate replication of non-data transactions -## such as table creation, rename, drop, and uploading of any closed WAL segments. -## Only set to `true` if your application is highly sensitive to network overhead. -## In most cases, tweak `cairo.wal.segment.rollover.size` and -## `replication.primary.throttle.window.duration` instead. -#replication.primary.throttle.non.data=false - -## During the upload each table's transaction log is chunked into smaller sized -## "parts". Each part defaults to 5000 transactions. Before compression, each -## transaction requires 28 bytes, and the last transaction log part is re-uploaded -## in full each time for each data upload. -## If you are heavily network constraint you may want to lower this (to say 2000), -## but this would in turn put more pressure on the object store with a larger -## amount of tiny object uploads (values lower than 1000 are never recommended). -## Note: This setting only applies to newly created tables. -#replication.primary.sequencer.part.txn.count=5000 - -## Max number of threads used to perform file compression operations before -## uploading to the object store. The default value is calculated as half the -## number of CPU cores. -#replication.primary.compression.threads= - -## Zstd compression level. Defaults to 1. Valid values are from -7 to 22. -## Where -7 is fastest and least compressed, and 22 is most CPU and memory intensive -## and most compressed. -#replication.primary.compression.level=1 - -## Whether to calculate and include a checksum with every uploaded artifact. -## By default this is is done only for services (object store implementations) -## that don't provide this themselves (typically, the filesystem). -## The other options are "never" and "always". -#replication.primary.checksum=service-dependent - -## Each time the primary instance upload a new version of the index, -## it extends the ownership time lease of the object store, ensuring no other -## primary instance may be started in its place. -## The keepalive interval determines how long to wait after a period of inactivity -## before triggering a new empty ownership-extending upload. -## This setting also determines how long the database should when migrating ownership -## to a new instance. -## The wait is calculated as `3 x $replication.primary.keepalive.interval`. -#replication.primary.keepalive.interval=10s - -## Each upload for a given table will trigger an update of the index. -## If a database has many actively replicating tables, this could trigger -## overwriting the index path on the object store many times per second. -## This can be an issue with Google Cloud Storage which has a hard-limit of -## of 1000 requests per second, per path. -## As such, we default this to a 1s grouping time window for for GCS, and no -## grouping time window for other object store providers. -#replication.primary.index.upload.throttle.interval= - -## During the uploading process WAL column files may have unused trailing data. -## The default is to skip reading/compressing/uploading this data. -#replication.primary.upload.truncated=true - -## Polling rate of a replica instance to check for new changes (milliseconds). -#replication.replica.poll.interval=1000 - -## Network buffer size (byte count) used when downloading data from the object store. -#replication.requests.buffer.size=32768 - -## How often to produce a summary of the replication progress for each table -## in the logs. The default is to do this once a minute. -## You may disable the summary by setting this parameter to 0. -#replication.summary.interval=1m - -## Enable per-table Prometheus metrics on replication progress. -## These are enabled by default. -#replication.metrics.per.table=true - -## How many times (number of HTTP Prometheus polled scrapes) should per-table -## replication metrics continue to be published for dropped tables. -## The default of 10 is designed to ensure that transaction progress -## for dropped tables is not accidentally missed. -## Consider raising if you scrape each database instance from multiple Prometheus -## instances. -#replication.metrics.dropped.table.poll.count=10 - -## Number of parallel requests to cap at when uploading or downloading data for a given -## iteration. This cap is used to perform partial progress when there is a large amount -## of outstanding uploads/downloads. -## The `fast` version is used by default for good throughput when everything is working -## well, while `slow` is used after the uploads/downloads encounter issues to allow -## progress in case of flaky networking conditions. -# replication.requests.max.batch.size.fast=64 -# replication.requests.max.batch.size.slow=2 - -## When the replication logic uploads, it times out and tries again if requests take too -## long. Since the timeout is dependent on the upload artifact size, we use a base timeout -## and a minimum throughput. The base timeout is of 10 seconds. -## The additional throughput-derived timeout is calculated from the size of the artifact -## and expressed as bytes per second. -#replication.requests.base.timeout=10s -#replication.requests.min.throughput=262144 - -### OIDC (QuestDB Enterprise Only) - -# Enables/Disables OIDC authentication. Once enabled few other -# configuration options must also be set. -#acl.oidc.enabled=false - -# Required: OIDC provider hostname -#acl.oidc.host= - -# Required: OIDC provider port number. -#acl.oidc.port=443 - -# Optional: OIDC provider host TLS setting; TLS is enabled by default. -#acl.oidc.tls.enabled=true - -# Optional: Enables QuestDB to validate TLS configuration, such as -# validity of TLS certificates. If you are working with self-signed -# certificates that you would like QuestDB to trust, disable this validations. -# Validation is strongly recommended in production environments. -#acl.oidc.tls.validation.enabled=true - -# Optional: path to keystore that contains certificate information of the -# OIDC provider. This is not required if your OIDC provider uses public CAs -#acl.oidc.tls.keystore.path= - -# Optional: keystore password, if the keystore is password protected. -#acl.oidc.tls.keystore.password= - -# Optional: OIDC provider HTTP request timeout in milliseconds -#acl.oidc.http.timeout=30000 - -# Required: OIDC provider client name. Typically OIDC provider will -# require QuestDB instance to become a "client" of the OIDC server. This -# procedure requires a "client" named entity to be created on OIDC server. -# Copy the name of the client to this configuration option -#acl.oidc.client.id= - -# Optional: OIDC authorization endpoint; the default value should work for Ping Identity Platform -#acl.oidc.authorization.endpoint=/as/authorization.oauth2 - -# Optional: OAUTH2 token endpoint; the default value should work for Ping Identity Platform -#acl.oidc.token.endpoint=/as/token.oauth2 - -# Optional: OIDC user info endpoint; the default value should work for Ping Identity Platform -#acl.oidc.userinfo.endpoint=/idp/userinfo.openid - -# Required: OIDC provider must include group array into user info response. Typically -# this is a JSON response that contains list of claim objects. Group claim is -# the name of the claim in that JSON object that contains an array of user group -# names. -#acl.oidc.groups.claim=groups - -# Optional: QuestDB instance will cache tokens for the specified TTL (millis) period before -# they are revalidated again with OIDC provider. It is a performance related setting to avoid -# otherwise stateless QuestDB instance contacting OIDC provider too frequently. -# If set to 0, the cache is disabled completely. -#acl.oidc.cache.ttl=30000 - -# Optional: QuestDB instance will cache the OIDC provider's public keys which can be used to check -# a JWT token's validity. QuestDB will reload the public keys after this expiry time to pickup -# any changes. It is a performance related setting to avoid contacting the OIDC provider too frequently. -#acl.oidc.public.keys.expiry=120000 - -# Log timestamp is generated in UTC, however, it can be written out as -# timezone of your choice. Timezone name can be either explicit UTC offset, -# such as "+08:00", "-10:00", "-07:45", "UTC+05" or timezone name "EST", "CET" etc -# https://www.joda.org/joda-time/timezones.html. Please be aware that timezone database, -# that is included with QuestDB distribution is changing from one release to the next. -# This is not something that we change, rather Java releases include timezone database -# updates. -#log.timestamp.timezone=UTC - -# Timestamp locale. You can use this to have timestamp written out using localised month -# names and day of week names. For example, for Portuguese use 'pt' locale. -#log.timestamp.locale=en - -# Format pattern used to write out log timestamps -#log.timestamp.format=yyyy-MM-ddTHH:mm:ss.SSSUUUz - -#query.within.latest.by.optimisation.enabled diff --git a/ci/confs/default/server.conf b/ci/confs/default/server.conf deleted file mode 100644 index dfe5419..0000000 --- a/ci/confs/default/server.conf +++ /dev/null @@ -1,1395 +0,0 @@ -# Comment or set to false to allow QuestDB to start even in the presence of config errors. -config.validation.strict=true - -# toggle whether worker should stop on error -#shared.worker.haltOnError=false - -# Number of threads in Network shared thread pool. Network thread pool used for handling HTTP, TCP, UDP and Postgres connections unless dedicated thread pools are configured. -#shared.network.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.network.worker.count" used for Network thread pool. By default, threads have no CPU affinity -#shared.network.worker.affinity= - -# Number of threads in Query shared thread pool. Query thread pool used for handling parallel queries, like parallel filters and group by queries. -#shared.query.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.query.worker.count" used for Query thread pool. By default, threads have no CPU affinity -#shared.query.worker.affinity= - -# Number of threads in Write shared thread pool. Write pool threads are used for running WAL Apply work load to merge data from WAL files into the table -#shared.write.worker.count=2 - -# Comma-delimited list of CPU ids, one per thread specified in "shared.write.worker.count" used for Write thread pool. By default, threads have no CPU affinity -#shared.write.worker.affinity= - -# Default number of worker threads in Network, Query and Write shared pools. Single value to configure all three pools sizes. -#shared.worker.count=2 - -# RAM usage limit, as a percentage of total system RAM. A zero value does not -# set any limit. The default is 90. -#ram.usage.limit.percent=90 - -# RAM usage limit, in bytes. A zero value (the default) does not set any limit. -# If both this and ram.usage.limit.percent are non-zero, the lower limit takes precedence. -#ram.usage.limit.bytes=0 - -# Repeats compatible migrations from the specified version. The default setting of 426 allows to upgrade and downgrade QuestDB in the range of versions from 6.2.0 to 7.0.2. -# If set to -1 start time improves but downgrades to versions below 7.0.2 and subsequent upgrades can lead to data corruption and crashes. -#cairo.repeat.migration.from.version=426 - -################ HTTP settings ################## - -# enable HTTP server -http.enabled=true - -# IP address and port of HTTP server -#http.net.bind.to=0.0.0.0:9000 - -# Uncomment to enable HTTP Basic authentication -#http.user=admin -#http.password=quest - -# Maximum time interval for the HTTP server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.net.accept.loop.timeout=500 - -#http.net.connection.limit=256 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#http.net.connection.hint=false - - -# Maximum HTTP connections that can be used for ILP ingestion using /write http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.ilp.connection.limit=-1 - -# Maximum HTTP connections that can be used for queries using /query http endpoint. This limit must be lower or equal to http.net.connection.limit -# Not restricted by default. Database restart is NOT required when this setting is changed -#http.json.query.connection.limit=-1 - -# Maximum HTTP connections that can be used for export using /exp http endpoint. This limit must be lower or equal to http.net.connection.limit -# Restricted to number or CPUs or 25% of overall http connections, whichever is lower, by default. Database restart is NOT required when this setting is changed -#http.export.connection.limit=-1 - -# Idle HTTP connection timeout in milliseconds. -#http.net.connection.timeout=5m - -#Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#http.net.connection.queue.timeout=5000 - -# SO_SNDBUF value, -1 = OS default -#http.net.connection.sndbuf=2m - -# SO_RCVBUF value, -1 = OS default -#http.net.connection.rcvbuf=2m - -# size of receive buffer on application side -#http.receive.buffer.size=1m - -# initial size of the connection pool -#http.connection.pool.initial.capacity=4 - -# initial size of the string pool shared by HttpHeaderParser and HttpMultipartContentParser -#http.connection.string.pool.capacity=128 - -# HeaderParser buffer size in bytes -#http.multipart.header.buffer.size=512 - -# how long code accumulates incoming data chunks for column and delimiter analysis -#http.multipart.idle.spin.count=10000 - -#http.request.header.buffer.size=64k - -#http.worker.count=0 -#http.worker.affinity= -#http.worker.haltOnError=false - -# size of send data buffer -#http.send.buffer.size=2m - -# sets the clock to always return zero -#http.frozen.clock=false - -#http.allow.deflate.before.send=false - -# HTTP session timeout -#http.session.timeout=30m - -## When you using SSH tunnel you might want to configure -## QuestDB HTTP server to switch to HTTP/1.0 - -## Set HTTP protocol version to HTTP/1.0 -#http.version=HTTP/1.1 -## Set server keep alive to 'false'. This will make server disconnect client after -## completion of each request -#http.server.keep.alive=true - -## When in HTTP/1.0 mode keep alive values must be 0 -#http.keep-alive.timeout=5 -#http.keep-alive.max=10000 - -#http.static.public.directory=public - -#http.text.date.adapter.pool.capacity=16 -#http.text.json.cache.limit=16384 -#http.text.json.cache.size=8192 -#http.text.max.required.delimiter.stddev=0.1222d -#http.text.max.required.line.length.stddev=0.8 -#http.text.metadata.string.pool.capacity=128 -#http.text.roll.buffer.limit=8216576 -#http.text.roll.buffer.size=1024 -#http.text.analysis.max.lines=1000 -#http.text.lexer.string.pool.capacity=64 -#http.text.timestamp.adapter.pool.capacity=64 -#http.text.utf8.sink.size=4096 - -#http.json.query.connection.check.frequency=1000000 - -# enables the query cache -#http.query.cache.enabled=true - -# sets the number of blocks for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.block.count= 8 * worker_count - -# sets the number of rows for the query cache. Cache capacity is number_of_blocks * number_of_rows. -#http.query.cache.row.count= 2 * worker_count - -# sets the /settings endpoint readonly -#http.settings.readonly=false - -#http.security.readonly=false -#http.security.max.response.rows=Long.MAX_VALUE - -# Context path for the Web Console. If other REST services remain on the -# default context paths they will move to the same context path as the Web Console. -# Exception is ILP HTTP services, which are not used by the Web Console. They will -# remain on their default context paths. When the default context paths are changed, -# moving the Web Console will not affect the configured paths. QuestDB will create -# a copy of those services on the paths used by the Web Console so the outcome is -# both the Web Console and the custom service are operational. -#http.context.web.console=/ - -# Context path of the file import service -#http.context.import=/imp -# This service is used by the import UI in the Web Console -#http.context.table.status=/chk - -# Context path of the SQL result CSV export service -#http.context.export=/exp - -# This service provides server-side settings to the Web Console -#http.context.settings=/settings - -# SQL execution service -#http.context.execute=/exec - -# Web Console specific service -#http.context.warnings=/warnings - -# ILP HTTP Services. These are not used by the Web Console -#http.context.ilp=/write,/api/v2/write -#http.context.ilp.ping=/ping - -# Custom HTTP redirect service. All redirects are 301 - Moved permanently -#http.redirect.count=1 -#http.redirect.1=/ -> /index.html - -# circuit breaker is a mechanism that interrupts query execution -# at present queries are interrupted when remote client disconnects or when execution takes too long -# and times out - -# circuit breaker is designed to be invoke continuously in a tight loop -# the throttle is a number of pin cycles before abort conditions are tested -#circuit.breaker.throttle=2000000 - -# buffer used by I/O dispatchers and circuit breakers to check the socket state, please do not change this value -# the check reads \r\n from the input stream and discards it since some HTTP clients send this as a keep alive in between requests -#net.test.connection.buffer.size=64 - -# max execution time for read-only query in seconds -# "insert" type of queries are not aborted unless they -# it is "insert as select", where select takes long time before producing rows for the insert -query.timeout=5s - -## HTTP MIN settings -## -## Use this port to health check QuestDB instance when it isn't desired to log these health check requests. This is sort of /dev/null for monitoring - -#http.min.enabled=true -#http.min.net.bind.to=0.0.0.0:9003 - -# When enabled, health check will return HTTP 500 if there were any unhandled errors since QuestDB instance start. -#http.pessimistic.health.check.enabled=false - -# Maximum time interval for the HTTP MIN server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#http.min.net.accept.loop.timeout=500 - -################ Cairo settings ################## - -# directory for storing db tables and metadata. this directory is inside the server root directory provided at startup -#cairo.root=db - -# how changes to table are flushed to disk upon commit - default: nosync. Choices: nosync, async (flush call schedules update, returns immediately), sync (waits for flush to complete) -#cairo.commit.mode=nosync - -# The amount of time server is allowed to be compounding transaction before physical commit is forced. -# Compounding of the transactions improves system's through but hurts latency. Reduce this value to -# reduce latency of data visibility. -#cairo.commit.latency=30s - -# number of types table creation or insertion will be attempted -#cairo.create.as.select.retry.count=5 - -# comma separated list of volume definitions, volume_alias -> absolute_path_to_existing_directory. -# volume alias can then be used in create table statement with IN VOLUME clause -#cairo.volumes= by default IN VOLUME is switched off, no volume definitions. - -# type of map uses. Options: 1. fast (speed at the expense of storage. this is the default option) 2. compact -#cairo.default.map.type=fast - -# when true, symbol values will be cached on Java heap -#cairo.default.symbol.cache.flag=false - -# when column type is SYMBOL this parameter specifies approximate capacity for symbol map. -# It should be equal to number of unique symbol values stored in the table and getting this -# value badly wrong will cause performance degradation. Must be power of 2 -#cairo.default.symbol.capacity=256 - -# number of attempts to open files -#cairo.file.operation.retry.count=30 - -# when DB is running in sync/async mode, how many 'steps' IDGenerators will pre-allocate and synchronize sync to disk -#cairo.id.generator.batch.step=512 - -# how often the writer maintenance job gets run -#cairo.idle.check.interval=5m - -# defines the number of latest partitions to keep open when returning a reader to the reader pool -#cairo.inactive.reader.max.open.partitions=128 - -# defines frequency in milliseconds with which the reader pool checks for inactive readers. -#cairo.inactive.reader.ttl=2m - -# defines frequency in milliseconds with which the writer pool checks for inactive readers. -#cairo.inactive.writer.ttl=10m - -# when true (default), TTL enforcement uses wall clock time to prevent accidental data loss -# when future timestamps are inserted. When false, TTL uses only the max timestamp in the table. -#cairo.ttl.use.wall.clock=true - -# approximation of number of rows for single index key, must be power of 2 -#cairo.index.value.block.size=256 - -# number of attempts to open swap file -#cairo.max.swap.file.count=30 - -# file permission for new directories -#cairo.mkdir.mode=509 - -# Time to wait before retrying writing into a table after a memory limit failure -#cairo.write.back.off.timeout.on.mem.pressure=4s - -# maximum file name length in chars. Affects maximum table name length and maximum column name length -#cairo.max.file.name.length=127 - -# minimum number of rows before allowing use of parallel indexation -#cairo.parallel.index.threshold=100000 - -# number of segments in the TableReader pool; each segment holds up to 32 readers -#cairo.reader.pool.max.segments=10 - -# timeout in milliseconds when attempting to get atomic memory snapshots, e.g. in BitmapIndexReaders -#cairo.spin.lock.timeout=1s - -# sets size of the CharacterStore -#cairo.character.store.capacity=1024 - -# Sets size of the CharacterSequence pool -#cairo.character.store.sequence.pool.capacity=64 - -# sets size of the Column pool in the SqlCompiler -#cairo.column.pool.capacity=4096 - -# size of the ExpressionNode pool in SqlCompiler -#cairo.expression.pool.capacity=8192 - -# load factor for all FastMaps -#cairo.fast.map.load.factor=0.7 - -# size of the JoinContext pool in SqlCompiler -#cairo.sql.join.context.pool.capacity=64 - -# size of FloatingSequence pool in GenericLexer -#cairo.lexer.pool.capacity=2048 - -# sets the key capacity in FastMap and CompactMap -#cairo.sql.map.key.capacity=2097152 - -# sets the key capacity in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.key.capacity=32 - -# number of map resizes in FastMap and CompactMap before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.map.max.resizes=2^31 - -# memory page size for FastMap and CompactMap -#cairo.sql.map.page.size=4m - -# memory page size in FastMap and CompactMap used in certain queries, e.g. SAMPLE BY -#cairo.sql.small.map.page.size=32k - -# memory max pages for CompactMap -#cairo.sql.map.max.pages=2^31 - -# sets the size of the QueryModel pool in the SqlCompiler -#cairo.model.pool.capacity=1024 - -# sets the maximum allowed negative value used in LIMIT clause in queries with filters -#cairo.sql.max.negative.limit=10000 - -# sets the memory page size for storing keys in LongTreeChain -#cairo.sql.sort.key.page.size=128k - -# max number of pages for storing keys in LongTreeChain before a resource limit exception is thrown -# cairo.sql.sort.key.max.pages=2^31 - -# sets the memory page size and max pages for storing values in LongTreeChain -#cairo.sql.sort.light.value.page.size=128k -#cairo.sql.sort.light.value.max.pages=2^31 - -# sets the memory page size and max pages of the slave chain in full hash joins -#cairo.sql.hash.join.value.page.size=16777216 -#cairo.sql.hash.join.value.max.pages=2^31 - -# sets the initial capacity for row id list used for latest by -#cairo.sql.latest.by.row.count=1000 - -# sets the memory page size and max pages of the slave chain in light hash joins -#cairo.sql.hash.join.light.value.page.size=128k -#cairo.sql.hash.join.light.value.max.pages=2^31 - -# number of rows to scan linearly before starting binary search in ASOF JOIN queries with no additional keys -#cairo.sql.asof.join.lookahead=10 - -# sets memory page size and max pages of file storing values in SortedRecordCursorFactory -#cairo.sql.sort.value.page.size=16777216 -#cairo.sql.sort.value.max.pages=2^31 - -# latch await timeout in nanoseconds for stealing indexing work from other threads -#cairo.work.steal.timeout.nanos=10000 - -# whether parallel indexation is allowed. Works in conjunction with cairo.parallel.index.threshold -#cairo.parallel.indexing.enabled=true - -# memory page size for JoinMetadata file -#cairo.sql.join.metadata.page.size=16384 - -# number of map resizes in JoinMetadata before a resource limit exception is thrown, each resize doubles the previous size -#cairo.sql.join.metadata.max.resizes=2^31 - -# size of PivotColumn pool in SqlParser -#cairo.sql.pivot.column.pool.capacity=64 - -# maximum number of columns PIVOT can produce (FOR value combinations × aggregates) -#cairo.sql.pivot.max.produced.columns=5000 - -# size of WindowColumn pool in SqlParser -#cairo.sql.window.column.pool.capacity=64 - -# sets the memory page size and max number of pages for records in window function -#cairo.sql.window.store.page.size=1m -#cairo.sql.window.store.max.pages=2^31 - -# sets the memory page size and max number of pages for row ids in window function -#cairo.sql.window.rowid.page.size=512k -#cairo.sql.window.rowid.max.pages=2^31 - -# sets the memory page size and max number of pages for keys in window function -#cairo.sql.window.tree.page.size=512k -#cairo.sql.window.tree.max.pages=2^31 - -# sets initial size of per-partition window function range frame buffer -#cairo.sql.window.initial.range.buffer.size=32 - -# batch size of non-atomic inserts for CREATE TABLE AS SELECT statements -#cairo.sql.create.table.model.batch.size=1000000 - -# Size of the pool for model objects, that underpin the "create table" parser. -# It is aimed at reducing allocations and it a performance setting. The -# number is aligned to the max concurrent "create table" requests the system -# will ever receive. If system receives more requests that this, it will just -# allocate more object and free them after use. -#cairo.create.table.column.model.pool.capacity=16 - -# size of RenameTableModel pool in SqlParser -#cairo.sql.rename.table.model.pool.capacity=16 - -# size of WithClauseModel pool in SqlParser -#cairo.sql.with.clause.model.pool.capacity=128 - -# size of CompileModel pool in SqlParser -#cairo.sql.compile.view.model.pool.capacity=8 - -# initial size of view lexer pool in SqlParser, used to parse SELECT statements of view definitions -# the max number of views used in a single query determines how many view lexers should be in the pool -#cairo.sql.view.lexer.pool.capacity=8 - -# size of InsertModel pool in SqlParser -#cairo.sql.insert.model.pool.capacity=64 - -# batch size of non-atomic inserts for INSERT INTO SELECT statements -#cairo.sql.insert.model.batch.size=1000000 - -# enables parallel GROUP BY execution; by default, parallel GROUP BY requires at least 4 shared worker threads to take place -#cairo.sql.parallel.groupby.enabled=true - -# merge queue capacity for parallel GROUP BY; used for parallel tasks that merge shard hash tables -#cairo.sql.parallel.groupby.merge.shard.queue.capacity= - -# threshold for parallel GROUP BY to shard the hash table holding the aggregates -#cairo.sql.parallel.groupby.sharding.threshold=10000 - -# enables statistics-based hash table pre-sizing in parallel GROUP BY -#cairo.sql.parallel.groupby.presize.enabled=true - -# maximum allowed hash table size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.capacity=100000000 - -# maximum allowed heap size for parallel GROUP BY hash table pre-sizing -#cairo.sql.parallel.groupby.presize.max.heap.size=1G - -# threshold for parallel ORDER BY + LIMIT execution on sharded GROUP BY hash table -#cairo.sql.parallel.groupby.topk.threshold=5000000 - -# queue capacity for parallel ORDER BY + LIMIT applied to sharded GROUP BY -#cairo.sql.parallel.groupby.topk.queue.capacity= - -# threshold for in-flight tasks for disabling work stealing during parallel SQL execution -# when the number of shared workers is less than 4x of this setting, work stealing is always enabled -#cairo.sql.parallel.work.stealing.threshold=16 - -# spin timeout in nanoseconds for adaptive work stealing strategy -# controls how long the query thread waits for worker threads to pick up tasks before stealing work back -#cairo.sql.parallel.work.stealing.spin.timeout=50000 - -# enables parallel read_parquet() SQL function execution; by default, parallel read_parquet() requires at least 4 shared worker threads to take place -#cairo.sql.parallel.read.parquet.enabled=true - -# capacity for Parquet page frame cache; larger values may lead to better ORDER BY and some other -# clauses performance at the cost of memory overhead -#cairo.sql.parquet.frame.cache.capacity=3 - -# default size for memory buffers in GROUP BY function native memory allocator -#cairo.sql.groupby.allocator.default.chunk.size=128K - -# maximum allowed native memory allocation for GROUP BY functions -#cairo.sql.groupby.allocator.max.chunk.size=4G - -# threshold in bytes for switching from single memory buffer hash table (unordered) to a hash table with separate heap for entries (ordered) -#cairo.sql.unordered.map.max.entry.size=32 - -## prevents stack overflow errors when evaluating complex nested SQLs -## the value is an approximate number of nested SELECT clauses. -#cairo.sql.window.max.recursion=128 - -## pre-sizes the internal data structure that stores active query executions -## the value is chosen automatically based on the number of threads in the shared worker pool -#cairo.sql.query.registry.pool.size= - -## window function buffer size in record counts -## pre-sizes buffer for every windows function execution to contain window records -#cairo.sql.analytic.initial.range.buffer.size=32 - -## enables quick and radix sort in order by, when applicable -#cairo.sql.orderby.sort.enabled=true - -## defines number of rows to use radix sort in order by -#cairo.sql.orderby.radix.sort.threshold=600 - -## enables the column alias to be generated from the expression -#cairo.sql.column.alias.expression.enabled=true - -## maximum length of generated column aliases -#cairo.sql.column.alias.generated.max.size=64 - -## initial capacity of string pool for preferences store and parser -#cairo.preferences.string.pool.capacity=64 - -## Flag to enable or disable symbol capacity auto-scaling. Auto-scaling means resizing -## symbol table data structures as the number of symbols in the table grows. Optimal sizing of -## these data structures ensures optimal ingres performance. -## -## By default, the auto-scaling is enabled. This is optimal. You may want to disable auto-scaling in case -## something goes wrong. -## -## Database restart is NOT required when this setting is changed, but `reload_config()` SQL should be executed. -#cairo.auto.scale.symbol.capacity=true - -## Symbol occupancy threshold after which symbol capacity is doubled. For example -## threshold of 0.8 means that occupancy have to reach 80% of capacity before capacity is increased -#cairo.auto.scale.symbol.capacity.threshold=0.8 - -#### SQL COPY - -# size of CopyModel pool in SqlParser -#cairo.sql.copy.model.pool.capacity=32 - -# size of buffer used when copying tables -#cairo.sql.copy.buffer.size=2m - -# name of file with user's set of date and timestamp formats -#cairo.sql.copy.formats.file=/text_loader.json - -# input root directory, where COPY command and read_parquet() function read files from -# relative paths are resolved against the server root directory -cairo.sql.copy.root=import - -# export root directory, where COPY .. to command write files to -# relative paths are resolved against the server root directory -#cairo.sql.copy.export.root=export - -# input work directory, where temporary import files are created, by default it's located in tmp directory inside the server root directory -#cairo.sql.copy.work.root=null - -# default max size of intermediate import file index chunk (100MB). Import shouldn't use more memory than worker_count * this . -#cairo.sql.copy.max.index.chunk.size=100M - -# Capacity of the internal queue used to split parallel copy SQL command into subtasks and execute them across shared worker threads. -# The default configuration should be suitable for importing files of any size. -#cairo.sql.copy.queue.capacity=32 - -# Capacity of the internal queue used to execute copy export SQL command. -#cairo.sql.copy.export.queue.capacity=32 - -# Frequency of logging progress when exporting data using COPY .. TO command to export to parquet format. 0 or negative value disables the logging. -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.copy.report.frequency.lines=50000 - -# Parquet version to use when exporting data using COPY .. TO command. Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.parquet.export.version=1 - -# Enable statistics collection in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files when exporting data using COPY .. TO command -#cairo.parquet.export.raw.array.encoding.enabled=false - -# Compression codec to use when exporting data using COPY .. TO command. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.parquet.export.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs when exporting data using COPY .. TO command -#cairo.parquet.export.compression.level=9 - -# Row group size in rows when exporting data using COPY .. TO command. 0 uses the default (512*512 rows) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.row.group.size=0 - -# Data page size in bytes when exporting data using COPY .. TO command. 0 uses the default (1024*1024 bytes) -# Database restart is NOT required when this setting is changed -#cairo.parquet.export.data.page.size=0 - -# Maximum time to wait for export to complete when using /exp HTTP endpoint. 0 means no timeout. -#http.export.timeout=300s - -# Parquet version to use for partition encoder (for storing partitions in parquet format). Valid values: 1 (PARQUET_1_0), 2 (PARQUET_2_LATEST) -#cairo.partition.encoder.parquet.version=1 - -# Enable statistics collection in Parquet files for partition encoder -#cairo.partition.encoder.parquet.statistics.enabled=true - -# Enable raw array encoding for repeated fields in Parquet files for partition encoder -#cairo.partition.encoder.parquet.raw.array.encoding.enabled=false - -# Compression codec to use for partition encoder. Valid values: -# UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW -#cairo.partition.encoder.parquet.compression.codec=ZSTD - -# Compression level for GZIP (0-9), BROTLI (0-11), or ZSTD (1-22) codecs for partition encoder -#cairo.partition.encoder.parquet.compression.level=9 - -# Row group size in rows for partition encoder -#cairo.partition.encoder.parquet.row.group.size=100000 - -# Data page size in bytes for partition encoder -#cairo.partition.encoder.parquet.data.page.size=1048576 - -# Number of days to retain records in import log table (sys.parallel_text_import_log). Old records get deleted on each import and server restart. -#cairo.sql.copy.log.retention.days=3 - -# output root directory for backups -#cairo.sql.backup.root=null - -# date format for backup directory -#cairo.sql.backup.dir.datetime.format=yyyy-MM-dd - -# name of temp directory used during backup -#cairo.sql.backup.dir.tmp.name=tmp - -# permission used when creating backup directories -#cairo.sql.backup.mkdir.mode=509 - -# suffix of the partition directory in detached root to indicate it is ready to be attached -#cairo.attach.partition.suffix=.attachable - -# Use file system "copy" operation instead of "hard link" when attaching partition from detached root. Set to true if detached root is on a different drive. -#cairo.attach.partition.copy=false - -# file permission used when creating detached directories -#cairo.detached.mkdir.mode=509 - -# sample by index query page size - max values returned in single scan -# 0 means to use symbol block capacity -# cairo.sql.sampleby.page.size=0 - -# sample by default alignment -# if true, sample by will default to SAMPLE BY (FILL) ALIGN TO CALENDAR -# if false, sample by will default to SAMPLE BY (FILL) ALIGN TO FIRST OBSERVATION -# cairo.sql.sampleby.default.alignment.calendar=true - -# sets the minimum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.min.rows=100000 - -# sets the maximum number of rows in page frames used in SQL queries -#cairo.sql.page.frame.max.rows=1000000 - -# sets the minimum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.min.rows=10000 - -# sets the maximum number of rows in small page frames used in SQL queries, primarily for window joins -#cairo.sql.small.page.frame.max.rows=100000 - -# sets the memory page size and max number of pages for memory used by rnd functions -# currently rnd_str() and rnd_symbol(), this could extend to other rnd functions in the future -#cairo.rnd.memory.page.size=8K -#cairo.rnd.memory.max.pages=128 - -# max length (in chars) of buffer used to store result of SQL functions, such as replace() or lpad() -#cairo.sql.string.function.buffer.max.size=1048576 - -# SQL JIT compiler mode. Options: -# 1. on (enable JIT and use vector instructions when possible; default value) -# 2. scalar (enable JIT and use scalar instructions only) -# 3. off (disable JIT) -#cairo.sql.jit.mode=on - -# sets the memory page size and max pages for storing IR for JIT compilation -#cairo.sql.jit.ir.memory.page.size=8K -#cairo.sql.jit.ir.memory.max.pages=8 - -# sets the memory page size and max pages for storing bind variable values for JIT compiled filter -#cairo.sql.jit.bind.vars.memory.page.size=4K -#cairo.sql.jit.bind.vars.memory.max.pages=8 - -# sets debug flag for JIT compilation; when enabled, assembly will be printed into stdout -#cairo.sql.jit.debug.enabled=false - -# Controls the maximum allowed IN list length before the JIT compiler will fall back to Java code -#cairo.sql.jit.max.in.list.size.threshold=10 - -#cairo.date.locale=en - -# Maximum number of uncommitted rows in TCP ILP -#cairo.max.uncommitted.rows=500000 - -# Minimum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MinLag is the lower limit -#cairo.o3.min.lag=1s - -# Maximum size of in-memory buffer in milliseconds. The buffer is allocated dynamically through analysing -# the shape of the incoming data, and o3MaxLag is the upper limit -#cairo.o3.max.lag=600000 - -# Memory page size per column for O3 operations. Please be aware O3 will use 2x of this RAM per column -#cairo.o3.column.memory.size=8M - -# Memory page size per column for O3 operations on System tables only -#cairo.system.o3.column.memory.size=256k - -# Number of partition expected on average, initial value for purge allocation job, extended in runtime automatically -#cairo.o3.partition.purge.list.initial.capacity=1 - -# mmap sliding page size that TableWriter uses to append data for each column -#cairo.writer.data.append.page.size=16M - -# mmap page size for mapping small files, such as _txn, _todo and _meta -# the default value is OS page size (4k Linux, 64K windows, 16k OSX M1) -# if you override this value it will be rounded to the nearest (greater) multiple of OS page size -#cairo.writer.misc.append.page.size=4k - -# mmap page size for appending index key data; key data is number of distinct symbol values times 4 bytes -#cairo.writer.data.index.key.append.page.size=512k - -# mmap page size for appending value data; value data are rowids, e.g. number of rows in partition times 8 bytes -#cairo.writer.data.index.value.append.page.size=16M - -# mmap sliding page size that TableWriter uses to append data for each column specifically for System tables -#cairo.system.writer.data.append.page.size=256k - -# File allocation page min size for symbol table files -#cairo.symbol.table.min.allocation.page.size=4k - -# File allocation page max size for symbol table files -#cairo.symbol.table.max.allocation.page.size=8M - -# Maximum wait timeout in milliseconds for ALTER TABLE SQL statement run via REST and PG Wire interfaces when statement execution is ASYNCHRONOUS -#cairo.writer.alter.busy.wait.timeout=500ms - -# Row count to check writer command queue after on busy writing (e.g. tick after X rows written) -#cairo.writer.tick.rows.count=1024 - -# Maximum writer ALTER TABLE and replication command capacity. Shared between all the tables -#cairo.writer.command.queue.capacity=32 - -# Sets flag to enable io_uring interface for certain disk I/O operations on newer Linux kernels (5.12+). -#cairo.iouring.enabled=true - -# Minimum O3 partition prefix size for which O3 partition split happens to avoid copying the large prefix -#cairo.o3.partition.split.min.size=50M - -# The number of O3 partition splits allowed for the last partitions. If the number of splits grows above this value, the splits will be squashed -#cairo.o3.last.partition.max.splits=20 - -################ Parallel SQL execution ################ - -# Sets flag to enable parallel SQL filter execution. JIT compilation takes place only when this setting is enabled. -#cairo.sql.parallel.filter.enabled=true - -# Sets the threshold for column pre-touch to be run as a part of the parallel SQL filter execution. The threshold defines ratio between the numbers of scanned and filtered rows. -#cairo.sql.parallel.filter.pretouch.threshold=0.05 - -# Sets the upper limit on the number of in-flight tasks for parallel query workers published by filter queries with LIMIT. -#cairo.sql.parallel.filter.dispatch.limit= - -# Sets flag to enable parallel ORDER BY + LIMIT SQL execution. -#cairo.sql.parallel.topk.enabled=true - -# Sets flag to enable parallel WINDOW JOIN SQL execution. -#cairo.sql.parallel.window.join.enabled=true - -# Shard reduce queue contention between SQL statements that are executed concurrently. -#cairo.page.frame.shard.count=4 - -# Reduce queue is used for data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.page.frame.reduce.queue.capacity= - -# Reduce queue is used for vectorized data processing and should be large enough to supply tasks for worker threads (shared worked pool). -#cairo.vector.aggregate.queue.capacity= - -# Initial row ID list capacity for each slot of the "reduce" queue. Larger values reduce memory allocation rate, but increase RSS size. -#cairo.page.frame.rowid.list.capacity=256 - -# Initial column list capacity for each slot of the "reduce" queue. Used by JIT-compiled filters. -#cairo.page.frame.column.list.capacity=16 - -# Initial object pool capacity for local "reduce" tasks. These tasks are used to avoid blocking query execution when the "reduce" queue is full. -#cairo.page.frame.task.pool.capacity=4 - -################ LINE settings ###################### - -#line.default.partition.by=DAY - -# Enable / Disable automatic creation of new columns in existing tables via ILP. When set to false overrides value of line.auto.create.new.tables to false -#line.auto.create.new.columns=true - -# Enable / Disable automatic creation of new tables via ILP. -#line.auto.create.new.tables=true - -# Enable / Disable printing problematic ILP messages in case of errors. -#line.log.message.on.error=true - -################ LINE UDP settings ################## - -#line.udp.bind.to=0.0.0.0:9009 -#line.udp.join=232.1.2.3 -#line.udp.commit.rate=1000000 -#line.udp.msg.buffer.size=2048 -#line.udp.msg.count=10000 -#line.udp.receive.buffer.size=8m -line.udp.enabled=true -#line.udp.own.thread.affinity=-1 -#line.udp.own.thread=false -#line.udp.unicast=false -#line.udp.commit.mode=nosync -#line.udp.timestamp=n - -######################### LINE TCP settings ############################### - -#line.tcp.enabled=true -#line.tcp.net.bind.to=0.0.0.0:9009 -#line.tcp.net.connection.limit=256 - -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if net.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#line.tcp.net.connection.hint=false - -# Idle TCP connection timeout in milliseconds. 0 means there is no timeout. -#line.tcp.net.connection.timeout=0 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached -#line.tcp.net.connection.queue.timeout=5000 - -# Maximum time interval for the ILP TCP endpoint to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#line.tcp.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#line.tcp.net.connection.rcvbuf=-1 - -#line.tcp.connection.pool.capacity=64 -#line.tcp.timestamp=n - -# TCP message buffer size -#line.tcp.msg.buffer.size=2048 - -# Max measurement size -#line.tcp.max.measurement.size=2048 - -# Max receive buffer size -#line.tcp.max.recv.buffer.size=1073741824 - -# Size of the queue between the IO jobs and the writer jobs, each queue entry represents a measurement -#line.tcp.writer.queue.capacity=128 - -# IO and writer job worker pool settings, 0 indicates the shared pool should be used -#line.tcp.writer.worker.count=0 -#line.tcp.writer.worker.affinity= -#line.tcp.writer.worker.yield.threshold=10 -#line.tcp.writer.worker.nap.threshold=7000 -#line.tcp.writer.worker.sleep.threshold=10000 -#line.tcp.writer.halt.on.error=false - -#line.tcp.io.worker.count=0 -#line.tcp.io.worker.affinity= -#line.tcp.io.worker.yield.threshold=10 -#line.tcp.io.worker.nap.threshold=7000 -#line.tcp.io.worker.sleep.threshold=10000 -#line.tcp.io.halt.on.error=false - -# Sets flag to disconnect TCP connection that sends malformed messages. -#line.tcp.disconnect.on.error=true - -# Commit lag fraction. Used to calculate commit interval for the table according to the following formula: -# commit_interval = commit_lag ∗ fraction -# The calculated commit interval defines how long uncommitted data will need to remain uncommitted. -#line.tcp.commit.interval.fraction=0.5 -# Default commit interval in milliseconds. Used when o3MinLag is set to 0. -#line.tcp.commit.interval.default=2000 - -# Maximum amount of time in between maintenance jobs in milliseconds, these will commit uncommitted data -#line.tcp.maintenance.job.interval=1000 -# Minimum amount of idle time before a table writer is released in milliseconds -#line.tcp.min.idle.ms.before.writer.release=500 - - -#line.tcp.symbol.cache.wait.before.reload=500ms - -######################### LINE HTTP settings ############################### - -#line.http.enabled=true - -#line.http.max.recv.buffer.size=1073741824 - -#line.http.ping.version=v2.7.4 - -################ PG Wire settings ################## - -#pg.enabled=true -#pg.net.bind.to=0.0.0.0:8812 -#pg.net.connection.limit=64 -# Windows OS might have a limit on TCP backlog size. Typically Windows 10 has max of 200. This -# means that even if active.connection.limit is set over 200 it wont be possible to have this many -# concurrent connections. To overcome this limitation Windows has an unreliable hack, which you can -# exercise with this flag. Do only set it if you actively try to overcome limit you have already -# experienced. Read more about SOMAXCONN_HINT here https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-listen -#pg.net.connection.hint=false - -# Connection idle timeout in milliseconds. Connections are closed by the server when this timeout lapses. -#pg.net.connection.timeout=300000 - -# Amount of time in milliseconds a connection can wait in the listen backlog queue before its refused. Connections will be aggressively removed from the backlog until the active connection limit is breached. -#pg.net.connection.queue.timeout=5m - -# Maximum time interval for the PG Wire server to accept connections greedily in a loop. -# The accept loop timeout is set in milliseconds. -#pg.net.accept.loop.timeout=500 - -# SO_RCVBUF value, -1 = OS default -#pg.net.connection.rcvbuf=-1 - -# SO_SNDBUF value, -1 = OS default -#pg.net.connection.sndbuf=-1 - -#pg.legacy.mode.enabled=false -#pg.character.store.capacity=4096 -#pg.character.store.pool.capacity=64 -#pg.connection.pool.capacity=64 -#pg.password=quest -#pg.user=admin -# Enables read-only mode for the pg wire protocol. In this mode data mutation queries are rejected. -#pg.security.readonly=false -#pg.readonly.password=quest -#pg.readonly.user=user -# Enables separate read-only user for the pg wire server. Data mutation queries are rejected for all connections opened by this user. -#pg.readonly.user.enabled=false -# enables select query cache -#pg.select.cache.enabled=true -# sets the number of blocks for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.block.count= 8 * worker_count -# sets the number of rows for the select query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.select.cache.row.count= 2 * worker_count -# enables insert query cache -#pg.insert.cache.enabled=true -# sets the number of blocks for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.block.count=4 -# sets the number of rows for the insert query cache. Cache capacity is number_of_blocks * number_of_rows. -#pg.insert.cache.row.count=4 -#pg.max.blob.size.on.query=512k -#pg.recv.buffer.size=1M -#pg.net.connection.rcvbuf=-1 -#pg.send.buffer.size=1M -#pg.net.connection.sndbuf=-1 -#pg.date.locale=en -#pg.worker.count=2 -#pg.worker.affinity=-1,-1; -#pg.halt.on.error=false -#pg.daemon.pool=true -#pg.binary.param.count.capacity=2 -# maximum number of prepared statements a single client can create at a time. This is to prevent clients from creating -# too many prepared statements and exhausting server resources. -#pg.named.statement.limit=10000 - -# if you are using insert batches of over 64 rows, you should increase this value to avoid memory resizes that -# might slow down inserts. Be careful though as this allocates objects on JavaHeap. Setting this value too large -# might kill the GC and server will be unresponsive on startup. -#pg.pipeline.capacity=64 - -################ Materialized View settings ################## - -# Enables SQL support and refresh job for materialized views. -#cairo.mat.view.enabled=true - -# When disabled, SQL executed by materialized view refresh job always runs single-threaded. -#cairo.mat.view.parallel.sql.enabled=true - -# Desired number of base table rows to be scanned by one query during materialized view refresh. -#cairo.mat.view.rows.per.query.estimate=1000000 - -# Maximum time interval used for a single step of materialized view refresh. For larger intervals single SAMPLE BY interval will be used as the step. -#cairo.mat.view.max.refresh.step=31536000000000us - -# Maximum number of times QuestDB will retry a materialized view refresh after a recoverable error. -#cairo.mat.view.max.refresh.retries=10 - -# Timeout for next refresh attempts after an out-of-memory error in a materialized view refresh, in milliseconds. -#cairo.mat.view.refresh.oom.retry.timeout=200 - -# Maximum number of rows inserted by the refresh job in a single transaction. -#cairo.mat.view.insert.as.select.batch.size=1000000 - -# Maximum number of time intervals from WAL data transactions cached per materialized view. -#cairo.mat.view.max.refresh.intervals=100 - -# Interval for periodical refresh intervals caching for materialized views. -#cairo.mat.view.refresh.intervals.update.period=15s - -# Number of parallel threads used to refresh materialized view data. -# Configuration options: -# - Default: Automatically calculated based on CPU core count -# - 0: Runs as a single instance on the shared worker pool -# - N: Uses N dedicated threads for parallel refreshing -#mat.view.refresh.worker.count= - -# Materialized View refresh worker pool configuration -#mat.view.refresh.worker.affinity= -#mat.view.refresh.worker.yield.threshold=1000 -#mat.view.refresh.worker.nap.threshold=7000 -#mat.view.refresh.worker.sleep.threshold=10000 -#mat.view.refresh.worker.haltOnError=false - -# Export worker pool configuration -#export.worker.affinity= -#export.worker.yield.threshold=1000 -#export.worker.nap.threshold=7000 -#export.worker.sleep.threshold=10000 -#export.worker.haltOnError=false - -################ WAL settings ################## - -# Enable support of creating and writing to WAL tables -#cairo.wal.supported=true - -# If set to true WAL becomes the default mode for newly created tables. Impacts table created from ILP and SQL if WAL / BYPASS WAL not specified -#cairo.wal.enabled.default=true - -# Parallel threads to apply WAL data to the table storage. By default it is equal to the CPU core count. -# When set to 0 WAL apply job will run as a single instance on shared worker pool. -#wal.apply.worker.count= - -# WAL apply pool configuration -#wal.apply.worker.affinity= -#wal.apply.worker.yield.threshold=1000 -#wal.apply.worker.nap.threshold=7000 -#wal.apply.worker.sleep.threshold=10000 -#wal.apply.worker.haltOnError=false - -# Period in ms of how often WAL applied files are cleaned up from the disk -#cairo.wal.purge.interval=30s - -# Row count of how many rows are written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.size` (whichever is first). -#cairo.wal.segment.rollover.row.count=200000 - -# Byte size of number of rows written to the same WAL segment before starting a new segment. -# Triggers in conjunction with `cairo.wal.segment.rollover.row.count` (whichever is first). -# By default this is 0 (disabled) unless `replication.role=primary` is set, then it is defaulted to 2MiB. -#cairo.wal.segment.rollover.size=0 - -# mmap sliding page size that WalWriter uses to append data for each column -#cairo.wal.writer.data.append.page.size=1M - -# mmap sliding page size that WalWriter uses to append events for each column -# this page size has performance impact on large number of small transactions, larger -# page will cope better. However, if the workload is that of small number of large transaction, -# this page size can be reduced. The optimal value should be established via ingestion benchmark. -#cairo.wal.writer.event.append.page.size=128k - -# Multiplier to cairo.max.uncommitted.rows to calculate the limit of rows that can kept invisible when writing -# to WAL table under heavy load, when multiple transactions are to be applied. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.squash.uncommitted.rows.multiplier=20.0 - -# Maximum size of data can be kept in WAL lag in bytes. -# Default is 75M and it means that once the limit is reached, 50M will be committed and 25M will be kept in the lag. -# It is used to reduce the number Out Of Order commits when Out Of Order commits are unavoidable by squashing multiple commits together. -# Setting it very low can increase Out Of Order commit frequency and decrease the throughput. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.size=75M - -# Maximum number of transactions to keep in O3 lag for WAL tables. Once the number is reached, full commit occurs. -# By default it is -1 and the limit does not apply, instead the size limit of cairo.wal.max.lag.size is used. -# No rows are kept in WAL lag when last committed transaction is processed. -#cairo.wal.max.lag.txn.count=-1 - -# When WAL apply job processes transactions this is the minimum number of transaction -# to look ahead and read metadata of before applying any of them. -#cairo.wal.apply.look.ahead.txn.count=20 - -# Part of WAL apply job fair factor. The amount of time job spends on single table -# before moving to the next one. -#cairo.wal.apply.table.time.quota=1s - -# number of segments in the WalWriter pool; each segment holds up to 16 writers -#cairo.wal.writer.pool.max.segments=10 - -# number of segments in the ViewWalWriter pool; each segment holds up to 16 writers -#cairo.view.wal.writer.pool.max.segments=4 - -# ViewWalWriter pool releases inactive writers after this TTL. In milliseconds. -#cairo.view.wal.inactive.writer.ttl=60000 - -# Maximum number of Segments to cache File descriptors for when applying from WAL to table storage. -# WAL segment files are cached to avoid opening and closing them on processing each commit. -# Ideally should be in line with average number of simultaneous connections writing to the tables. -#cairo.wal.max.segment.file.descriptors.cache=30 - -# When disabled, SQL executed by WAL apply job always runs single-threaded. -#cairo.wal.apply.parallel.sql.enabled=true - -################ Telemetry settings ################## - -# Telemetry events are used to identify components of QuestDB that are being -# used. They never identify user nor data stored in QuestDB. All events can be -# viewed via `select * from telemetry`. Switching telemetry off will stop all -# events from being collected. After telemetry is switched off, telemetry -# tables can be dropped. - -telemetry.enabled=false -#telemetry.queue.capacity=512 -#telemetry.hide.tables=true -#telemetry.table.ttl.weeks=4 -#telemetry.event.throttle.interval=1m - -################ Metrics settings ################## - -#metrics.enabled=false - -############# Query Tracing settings ############### - -#query.tracing.enabled=false - -################ Logging settings ################## - -# enable or disable 'exe' log messages written when query execution begins -#log.sql.query.progress.exe=true - - -################ Enterprise configuration options ################## -### Please visit https://questdb.io/enterprise/ for more information -#################################################################### - - -#acl.enabled=false -#acl.entity.name.max.length=255 -#acl.admin.user.enabled=true -#acl.admin.user=admin -#acl.admin.password=quest - -## 10 seconds -#acl.rest.token.refresh.threshold=10 -#acl.sql.permission.model.pool.capacity=32 -#acl.password.hash.iteration.count=100000 - -#acl.basic.auth.realm.enabled=false - -#cold.storage.enabled=false -#cold.storage.object.store= - -#tls.enabled=false -#tls.cert.path= -#tls.private.key.path= - -## Defaults to the global TLS settings -## including enable/disable flag and paths -## Use these configuration values to separate -## min-http server TLS configuration from global settings -#http.min.tls.enabled= -#http.min.tls.cert.path= -#http.min.tls.private.key.path= - -#http.tls.enabled=false -#http.tls.cert.path= -#http.tls.private.key.path= - -#line.tcp.tls.enabled= -#line.tcp.tls.cert.path= -#line.tcp.tls.private.key.path= - -#pg.tls.enabled= -#pg.tls.cert.path= -#pg.tls.private.key.path= - -## The number of threads dedicated for async IO operations (e.g. network activity) in native code. -#native.async.io.threads= - -## The number of threads dedicated for blocking IO operations (e.g. file access) in native code. -#native.max.blocking.threads= - -### Replication (QuestDB Enterprise Only) - -# Possible roles are "primary", "replica", or "none" -# primary - read/write node -# replica - read-only node, consuming data from PRIMARY -# none - replication is disabled -#replication.role=none - -## Object-store specific string. -## AWS S3 example: -## s3::bucket=${BUCKET_NAME};root=${DB_INSTANCE_NAME};region=${AWS_REGION};access_key_id=${AWS_ACCESS_KEY};secret_access_key=${AWS_SECRET_ACCESS_KEY}; -## Azure Blob example: -## azblob::endpoint=https://${STORE_ACCOUNT}.blob.core.windows.net;container={BLOB_CONTAINER};root=${DB_INSTANCE_NAME};account_name=${STORE_ACCOUNT};account_key=${STORE_KEY}; -## Filesystem: -## fs::root=/nfs/path/to/dir/final;atomic_write_dir=/nfs/path/to/dir/scratch; -#replication.object.store= - -## Limits the number of concurrent requests to the object store. -## Defaults to 128 -#replication.requests.max.concurrent=128 - -## Maximum number of times to retry a failed object store request before -## logging an error and reattempting later after a delay. -#replication.requests.retry.attempts=3 - -## Delay between the retry attempts (milliseconds) -#replication.requests.retry.interval=200 - -## The time window grouping multiple transactions into a replication batch (milliseconds). -## Smaller time windows use more network traffic. -## Larger time windows increase the replication latency. -## Works in conjunction with `replication.primary.throttle.non.data`. -#replication.primary.throttle.window.duration=10000 - -## Set to `false` to allow immediate replication of non-data transactions -## such as table creation, rename, drop, and uploading of any closed WAL segments. -## Only set to `true` if your application is highly sensitive to network overhead. -## In most cases, tweak `cairo.wal.segment.rollover.size` and -## `replication.primary.throttle.window.duration` instead. -#replication.primary.throttle.non.data=false - -## During the upload each table's transaction log is chunked into smaller sized -## "parts". Each part defaults to 5000 transactions. Before compression, each -## transaction requires 28 bytes, and the last transaction log part is re-uploaded -## in full each time for each data upload. -## If you are heavily network constraint you may want to lower this (to say 2000), -## but this would in turn put more pressure on the object store with a larger -## amount of tiny object uploads (values lower than 1000 are never recommended). -## Note: This setting only applies to newly created tables. -#replication.primary.sequencer.part.txn.count=5000 - -## Max number of threads used to perform file compression operations before -## uploading to the object store. The default value is calculated as half the -## number of CPU cores. -#replication.primary.compression.threads= - -## Zstd compression level. Defaults to 1. Valid values are from -7 to 22. -## Where -7 is fastest and least compressed, and 22 is most CPU and memory intensive -## and most compressed. -#replication.primary.compression.level=1 - -## Whether to calculate and include a checksum with every uploaded artifact. -## By default this is is done only for services (object store implementations) -## that don't provide this themselves (typically, the filesystem). -## The other options are "never" and "always". -#replication.primary.checksum=service-dependent - -## Each time the primary instance upload a new version of the index, -## it extends the ownership time lease of the object store, ensuring no other -## primary instance may be started in its place. -## The keepalive interval determines how long to wait after a period of inactivity -## before triggering a new empty ownership-extending upload. -## This setting also determines how long the database should when migrating ownership -## to a new instance. -## The wait is calculated as `3 x $replication.primary.keepalive.interval`. -#replication.primary.keepalive.interval=10s - -## Each upload for a given table will trigger an update of the index. -## If a database has many actively replicating tables, this could trigger -## overwriting the index path on the object store many times per second. -## This can be an issue with Google Cloud Storage which has a hard-limit of -## of 1000 requests per second, per path. -## As such, we default this to a 1s grouping time window for for GCS, and no -## grouping time window for other object store providers. -#replication.primary.index.upload.throttle.interval= - -## During the uploading process WAL column files may have unused trailing data. -## The default is to skip reading/compressing/uploading this data. -#replication.primary.upload.truncated=true - -## Polling rate of a replica instance to check for new changes (milliseconds). -#replication.replica.poll.interval=1000 - -## Network buffer size (byte count) used when downloading data from the object store. -#replication.requests.buffer.size=32768 - -## How often to produce a summary of the replication progress for each table -## in the logs. The default is to do this once a minute. -## You may disable the summary by setting this parameter to 0. -#replication.summary.interval=1m - -## Enable per-table Prometheus metrics on replication progress. -## These are enabled by default. -#replication.metrics.per.table=true - -## How many times (number of HTTP Prometheus polled scrapes) should per-table -## replication metrics continue to be published for dropped tables. -## The default of 10 is designed to ensure that transaction progress -## for dropped tables is not accidentally missed. -## Consider raising if you scrape each database instance from multiple Prometheus -## instances. -#replication.metrics.dropped.table.poll.count=10 - -## Number of parallel requests to cap at when uploading or downloading data for a given -## iteration. This cap is used to perform partial progress when there is a large amount -## of outstanding uploads/downloads. -## The `fast` version is used by default for good throughput when everything is working -## well, while `slow` is used after the uploads/downloads encounter issues to allow -## progress in case of flaky networking conditions. -# replication.requests.max.batch.size.fast=64 -# replication.requests.max.batch.size.slow=2 - -## When the replication logic uploads, it times out and tries again if requests take too -## long. Since the timeout is dependent on the upload artifact size, we use a base timeout -## and a minimum throughput. The base timeout is of 10 seconds. -## The additional throughput-derived timeout is calculated from the size of the artifact -## and expressed as bytes per second. -#replication.requests.base.timeout=10s -#replication.requests.min.throughput=262144 - -### OIDC (QuestDB Enterprise Only) - -# Enables/Disables OIDC authentication. Once enabled few other -# configuration options must also be set. -#acl.oidc.enabled=false - -# Required: OIDC provider hostname -#acl.oidc.host= - -# Required: OIDC provider port number. -#acl.oidc.port=443 - -# Optional: OIDC provider host TLS setting; TLS is enabled by default. -#acl.oidc.tls.enabled=true - -# Optional: Enables QuestDB to validate TLS configuration, such as -# validity of TLS certificates. If you are working with self-signed -# certificates that you would like QuestDB to trust, disable this validations. -# Validation is strongly recommended in production environments. -#acl.oidc.tls.validation.enabled=true - -# Optional: path to keystore that contains certificate information of the -# OIDC provider. This is not required if your OIDC provider uses public CAs -#acl.oidc.tls.keystore.path= - -# Optional: keystore password, if the keystore is password protected. -#acl.oidc.tls.keystore.password= - -# Optional: OIDC provider HTTP request timeout in milliseconds -#acl.oidc.http.timeout=30000 - -# Required: OIDC provider client name. Typically OIDC provider will -# require QuestDB instance to become a "client" of the OIDC server. This -# procedure requires a "client" named entity to be created on OIDC server. -# Copy the name of the client to this configuration option -#acl.oidc.client.id= - -# Optional: OIDC authorization endpoint; the default value should work for Ping Identity Platform -#acl.oidc.authorization.endpoint=/as/authorization.oauth2 - -# Optional: OAUTH2 token endpoint; the default value should work for Ping Identity Platform -#acl.oidc.token.endpoint=/as/token.oauth2 - -# Optional: OIDC user info endpoint; the default value should work for Ping Identity Platform -#acl.oidc.userinfo.endpoint=/idp/userinfo.openid - -# Required: OIDC provider must include group array into user info response. Typically -# this is a JSON response that contains list of claim objects. Group claim is -# the name of the claim in that JSON object that contains an array of user group -# names. -#acl.oidc.groups.claim=groups - -# Optional: QuestDB instance will cache tokens for the specified TTL (millis) period before -# they are revalidated again with OIDC provider. It is a performance related setting to avoid -# otherwise stateless QuestDB instance contacting OIDC provider too frequently. -# If set to 0, the cache is disabled completely. -#acl.oidc.cache.ttl=30000 - -# Optional: QuestDB instance will cache the OIDC provider's public keys which can be used to check -# a JWT token's validity. QuestDB will reload the public keys after this expiry time to pickup -# any changes. It is a performance related setting to avoid contacting the OIDC provider too frequently. -#acl.oidc.public.keys.expiry=120000 - -# Log timestamp is generated in UTC, however, it can be written out as -# timezone of your choice. Timezone name can be either explicit UTC offset, -# such as "+08:00", "-10:00", "-07:45", "UTC+05" or timezone name "EST", "CET" etc -# https://www.joda.org/joda-time/timezones.html. Please be aware that timezone database, -# that is included with QuestDB distribution is changing from one release to the next. -# This is not something that we change, rather Java releases include timezone database -# updates. -#log.timestamp.timezone=UTC - -# Timestamp locale. You can use this to have timestamp written out using localised month -# names and day of week names. For example, for Portuguese use 'pt' locale. -#log.timestamp.locale=en - -# Format pattern used to write out log timestamps -#log.timestamp.format=yyyy-MM-ddTHH:mm:ss.SSSUUUz - -#query.within.latest.by.optimisation.enabled diff --git a/ci/questdb_start.yaml b/ci/questdb_start.yaml deleted file mode 100644 index 5799095..0000000 --- a/ci/questdb_start.yaml +++ /dev/null @@ -1,76 +0,0 @@ -parameters: - - name: configPath - type: string - -steps: - - bash: | - mkdir -p questdb-data - cp -r ${{ parameters.configPath }} questdb-data/conf - - mkdir -p logs - - # Start QuestDB in the background - java -p questdb/core/target/questdb-*-SNAPSHOT.jar -m io.questdb/io.questdb.ServerMain -d questdb-data > logs/questdb.log 2>&1 & - echo $! > questdb.pid - - # Wait for QuestDB to start - echo "Waiting for QuestDB to start..." - for i in {1..30}; do - if grep -q "server-main enjoy" logs/questdb.log; then - echo "Server started." - break - else - echo "Waiting..." - sleep 2 - fi - done - - if ! grep -q "server-main enjoy" logs/questdb.log; then - echo "QuestDB failed to start:" - cat logs/questdb.log - exit 1 - fi - displayName: "Start QuestDB server (non-Windows)" - condition: ne(variables['Agent.OS'], 'Windows_NT') - - pwsh: | - New-Item -ItemType Directory -Force -Path questdb-data | Out-Null - New-Item -ItemType Directory -Force -Path logs | Out-Null - Copy-Item -Recurse -Force -Path "${{ parameters.configPath }}" -Destination "questdb-data/conf" - - $jar = Get-ChildItem -Path "questdb/core/target" -Filter "questdb-*-SNAPSHOT.jar" | Select-Object -First 1 - if (-not $jar) { - Write-Host "QuestDB jar not found in questdb/core/target" - exit 1 - } - - $proc = Start-Process -FilePath "java" -ArgumentList @( - "-p", $jar.FullName, - "-m", "io.questdb/io.questdb.ServerMain", - "-d", "questdb-data" - ) -RedirectStandardOutput "logs/questdb.log" -RedirectStandardError "logs/questdb.err" -PassThru - - $proc.Id | Out-File -FilePath questdb.pid -Encoding ascii - - Write-Host "Waiting for QuestDB to start..." - $started = $false - for ($i = 0; $i -lt 30; $i++) { - if (Test-Path "logs/questdb.log") { - if (Select-String -Path "logs/questdb.log" -Pattern "server-main enjoy" -Quiet) { - Write-Host "Server started." - $started = $true - break - } - } - Write-Host "Waiting..." - Start-Sleep -Seconds 2 - } - - if (-not $started) { - Write-Host "QuestDB failed to start:" - if (Test-Path "logs/questdb.log") { - Get-Content "logs/questdb.log" - } - exit 1 - } - displayName: "Start QuestDB server (Windows)" - condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/ci/questdb_stop.yaml b/ci/questdb_stop.yaml deleted file mode 100644 index 5b56935..0000000 --- a/ci/questdb_stop.yaml +++ /dev/null @@ -1,39 +0,0 @@ -steps: - - bash: | - echo "Stopping QuestDB server..." - kill -TERM "$(cat questdb.pid)" || true - rm -f questdb.pid - rm -rf questdb-data - displayName: "Stop QuestDB server (non-Windows)" - condition: and(always(), ne(variables['Agent.OS'], 'Windows_NT')) - - pwsh: | - Write-Host "Stopping QuestDB server..." - if (Test-Path "questdb.pid") { - $questdbPid = Get-Content "questdb.pid" | Select-Object -First 1 - if ($questdbPid) { - Stop-Process -Id $questdbPid -ErrorAction SilentlyContinue - try { - Wait-Process -Id $questdbPid -Timeout 30 -ErrorAction Stop - } catch { - Stop-Process -Id $questdbPid -Force -ErrorAction SilentlyContinue - } - } - Remove-Item -Force "questdb.pid" -ErrorAction SilentlyContinue - } - if (Test-Path "questdb-data") { - $deleted = $false - for ($i = 0; $i -lt 10; $i++) { - try { - Remove-Item -Recurse -Force "questdb-data" -ErrorAction Stop - $deleted = $true - break - } catch { - Start-Sleep -Seconds 1 - } - } - if (-not $deleted) { - Remove-Item -Recurse -Force "questdb-data" - } - } - displayName: "Stop QuestDB server (Windows)" - condition: and(always(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/ci/run_client_tests.yaml b/ci/run_client_tests.yaml index e8e3216..11253cd 100644 --- a/ci/run_client_tests.yaml +++ b/ci/run_client_tests.yaml @@ -1,11 +1,4 @@ -parameters: - - name: configPath - type: string - steps: - - template: questdb_start.yaml - parameters: - configPath: ${{ parameters.configPath }} - task: Maven@3 displayName: "Run Client Tests" inputs: @@ -25,4 +18,3 @@ steps: inputs: pathToPublish: $(ARCHIVED_LOGS) artifactName: MavenFailedTestsLogs - - template: questdb_stop.yaml diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 00c0f8c..f1c8a3f 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -7,8 +7,6 @@ pr: variables: ARCHIVED_LOGS: "$(Build.ArtifactStagingDirectory)/questdb-$(Build.SourceBranchName)-$(Build.SourceVersion)-$(System.StageAttempt)-$(Agent.OS).zip" - QUESTDB_RUNNING: true - QUESTDB_ILP_TCP_AUTH_ENABLE: false stages: - stage: Validate @@ -111,12 +109,4 @@ stages: jdkVersionOption: "default" options: "-DskipTests -Pbuild-web-console --batch-mode" - template: run_client_tests.yaml - parameters: - configPath: "ci/confs/default" - - bash: | - echo "##vso[task.setvariable variable=QUESTDB_ILP_TCP_AUTH_ENABLE]true" - displayName: "Enable ILP TCP Auth" - - template: run_client_tests.yaml - parameters: - configPath: "ci/confs/authenticated" - template: run_oss_tests.yaml diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java deleted file mode 100644 index e984f19..0000000 --- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java +++ /dev/null @@ -1,674 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test; - -import io.questdb.client.std.Numbers; -import io.questdb.client.std.str.StringSink; -import io.questdb.client.std.str.Utf16Sink; -import io.questdb.client.test.tools.TestUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.io.InputStream; -import java.sql.Array; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.JDBCType; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -import static io.questdb.client.std.Numbers.hexDigits; -import static io.questdb.client.test.tools.TestUtils.assertEventually; -import static java.time.temporal.ChronoField.*; - -public class AbstractQdbTest extends AbstractTest { - - protected static final StringSink sink = new StringSink(); - private static final DateTimeFormatter DATE_TIME_FORMATTER; - // Configuration defaults - private static final String DEFAULT_HOST = "127.0.0.1"; - private static final int DEFAULT_HTTP_PORT = 9000; - private static final String DEFAULT_PG_PASSWORD = "quest"; - private static final int DEFAULT_PG_PORT = 8812; - private static final String DEFAULT_PG_USER = "admin"; - // Table name counter for uniqueness - private static final AtomicLong TABLE_NAME_COUNTER = new AtomicLong(System.currentTimeMillis()); - // Shared PostgreSQL connection - private static Connection pgConnection; - - /** - * Print the output of a SQL query to TSV format. - */ - public static long printToSink(StringSink sink, ResultSet rs) throws SQLException { - // dump metadata - ResultSetMetaData metaData = rs.getMetaData(); - final int columnCount = metaData.getColumnCount(); - for (int i = 0; i < columnCount; i++) { - if (i > 0) { - sink.put('\t'); - } - - sink.put(metaData.getColumnName(i + 1)); - } - sink.put('\n'); - - Timestamp timestamp; - long rows = 0; - while (rs.next()) { - rows++; - for (int i = 1; i <= columnCount; i++) { - if (i > 1) { - sink.put('\t'); - } - switch (JDBCType.valueOf(metaData.getColumnType(i))) { - case VARCHAR: - case NUMERIC: - String stringValue = rs.getString(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(stringValue); - } - break; - case INTEGER: - int intValue = rs.getInt(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(intValue); - } - break; - case DOUBLE: - double doubleValue = rs.getDouble(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(doubleValue); - } - break; - case TIMESTAMP: - timestamp = rs.getTimestamp(i); - if (timestamp == null) { - sink.put("null"); - } else { - sink.put(DATE_TIME_FORMATTER.format(timestamp.toLocalDateTime())); - } - break; - case REAL: - float floatValue = rs.getFloat(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(floatValue); - } - break; - case SMALLINT: - sink.put(rs.getShort(i)); - break; - case BIGINT: - long longValue = rs.getLong(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(longValue); - } - break; - case CHAR: - String strValue = rs.getString(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(strValue.charAt(0)); - } - break; - case BIT: - sink.put(rs.getBoolean(i)); - break; - case TIME: - case DATE: - timestamp = rs.getTimestamp(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(DATE_TIME_FORMATTER.format(timestamp.toLocalDateTime())); - } - break; - case BINARY: - InputStream stream = rs.getBinaryStream(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - toSink(stream, sink); - } - break; - case OTHER: - Object object = rs.getObject(i); - if (rs.wasNull()) { - sink.put("null"); - } else { - sink.put(object.toString()); - } - break; - case ARRAY: - Array array = rs.getArray(i); - if (array == null) { - sink.put("null"); - } else { - writeArrayContent(sink, array.getArray()); - } - break; - default: - assert false; - } - } - sink.put('\n'); - } - return rows; - } - - @BeforeClass - public static void setUpStatic() { - AbstractTest.setUpStatic(); - if (getQuestDBRunning()) { - System.err.printf("CLEANING UP TEST TABLES%n"); - // Cleanup all test tables before starting tests - try (Connection conn = getPgConnection(); - Statement readStmt = conn.createStatement(); - Statement stmt = conn.createStatement(); - ResultSet rs = readStmt - .executeQuery("SELECT table_name FROM tables() WHERE table_name LIKE 'test_%'")) { - while (rs.next()) { - String tableName = rs.getString(1); - try { - stmt.execute(String.format("DROP TABLE IF EXISTS '%s'", tableName)); - LOG.info("Dropped test table: {}", tableName); - } catch (SQLException e) { - LOG.warn("Failed to drop test table {}: {}", tableName, e.getMessage()); - } - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - } - - @AfterClass - public static void tearDownStatic() { - closePgConnection(); - } - - @Before - public void setUp() { - super.setUp(); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); - } - - private static void toSink(InputStream is, Utf16Sink sink) { - // limit what we print - byte[] bb = new byte[1]; - int i = 0; - try { - while (is.read(bb) > 0) { - byte b = bb[0]; - if (i > 0) { - if ((i % 16) == 0) { - sink.put('\n'); - Numbers.appendHexPadded(sink, i); - } - } else { - Numbers.appendHexPadded(sink, i); - } - sink.putAscii(' '); - - final int v; - if (b < 0) { - v = 256 + b; - } else { - v = b; - } - - if (v < 0x10) { - sink.putAscii('0'); - sink.putAscii(hexDigits[b]); - } else { - sink.putAscii(hexDigits[v / 0x10]); - sink.putAscii(hexDigits[v % 0x10]); - } - - i++; - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static void writeArrayContent(StringSink sink, Object array) { - if (array == null) { - sink.put("null"); - return; - } - if (!array.getClass().isArray()) { - if (array instanceof Number) { - if (array instanceof Double) { - double d = ((Number) array).doubleValue(); - if (Numbers.isNull(d)) { - sink.put("null"); - } else { - sink.put(d); - } - } - if (array instanceof Float) { - float f = ((Number) array).floatValue(); - if (Numbers.isNull(f)) { - sink.put("null"); - } else { - sink.put(f); - } - } - if (array instanceof Long) { - long l = ((Number) array).longValue(); - if (l == Numbers.LONG_NULL) { - sink.put("null"); - } else { - sink.put(l); - } - } - } else if (array instanceof Boolean) { - sink.put((Boolean) array); - } else { - sink.put(array.toString()); - } - return; - } - - sink.put('{'); - int length = java.lang.reflect.Array.getLength(array); - for (int i = 0; i < length; i++) { - Object element = java.lang.reflect.Array.get(array, i); - writeArrayContent(sink, element); - - if (i < length - 1) { - sink.put(','); - } - } - sink.put('}'); - } - - /** - * Close the shared PostgreSQL connection. - */ - protected static synchronized void closePgConnection() { - if (pgConnection != null) { - try { - pgConnection.close(); - LOG.info("Closed PostgreSQL connection"); - } catch (SQLException e) { - LOG.warn("Error closing PostgreSQL connection", e); - } finally { - pgConnection = null; - } - } - } - - /** - * Get configuration value from environment variable or system property. - * Environment variables take precedence over system properties. - */ - protected static String getConfig(String envKey, String sysPropKey, String defaultValue) { - String value = System.getenv(envKey); - if (value != null && !value.isEmpty()) { - return value; - } - return System.getProperty(sysPropKey, defaultValue); - } - - protected static boolean getConfigBool(String envKey, String sysPropKey, boolean defaultValue) { - String value = getConfig(envKey, sysPropKey, null); - if (value != null) { - return Boolean.parseBoolean(value); - } - return defaultValue; - } - - protected static int getConfigInt(String envKey, String sysPropKey, int defaultValue) { - String value = getConfig(envKey, sysPropKey, null); - if (value != null) { - try { - return Integer.parseInt(value); - } catch (NumberFormatException e) { - LOG.warn("Invalid integer value for {}/{}: {}, using default: {}", - envKey, sysPropKey, value, defaultValue); - } - } - return defaultValue; - } - - /** - * Get HTTP port. - */ - protected static int getHttpPort() { - return getConfigInt("QUESTDB_HTTP_PORT", "questdb.http.port", DEFAULT_HTTP_PORT); - } - - /** - * Get or create the shared PostgreSQL connection. - */ - protected static synchronized Connection getPgConnection() throws SQLException { - if (pgConnection == null || pgConnection.isClosed()) { - pgConnection = initPgConnection(); - } - return pgConnection; - } - - /** - * Get PostgreSQL password. - */ - protected static String getPgPassword() { - return getConfig("QUESTDB_PG_PASSWORD", "questdb.pg.password", DEFAULT_PG_PASSWORD); - } - - /** - * Get PostgreSQL wire protocol port. - */ - protected static int getPgPort() { - return getConfigInt("QUESTDB_PG_PORT", "questdb.pg.port", DEFAULT_PG_PORT); - } - - /** - * Get PostgreSQL user. - */ - protected static String getPgUser() { - return getConfig("QUESTDB_PG_USER", "questdb.pg.user", DEFAULT_PG_USER); - } - - /** - * Get whether a QuestDB instance is running locally. - */ - protected static boolean getQuestDBRunning() { - return getConfigBool("QUESTDB_RUNNING", "questdb.running", false); - } - - /** - * Get QuestDB host address. - */ - protected static String getQuestDbHost() { - return getConfig("QUESTDB_HOST", "questdb.host", DEFAULT_HOST); - } - - /** - * Initialize a new PostgreSQL connection to QuestDB. - */ - protected static Connection initPgConnection() throws SQLException { - String host = getQuestDbHost(); - int port = getPgPort(); - String user = getPgUser(); - String password = getPgPassword(); - - String url = String.format("jdbc:postgresql://%s:%d/qdb?sslmode=disable", host, port); - LOG.info("Connecting to QuestDB via PostgreSQL wire protocol: {}", url); - - return DriverManager.getConnection(url, user, password); - } - - /** - * Assert that SQL query results match expected TSV, polling until they do or - * timeout is reached. - */ - protected void assertSqlEventually(CharSequence expected, String sql) throws Exception { - assertEventually(() -> { - try (Statement statement = getPgConnection().createStatement(); - ResultSet rs = statement.executeQuery(sql)) { - sink.clear(); - printToSink(sink, rs); - TestUtils.assertEquals(expected, sink); - } catch (SQLException e) { - throw new RuntimeException(e); - } - }, 5); - } - - /** - * Assert that table exists, polling until it does or timeout is reached. - */ - protected void assertTableExistsEventually(CharSequence tableName) throws Exception { - assertEventually(() -> { - try (Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery( - String.format("SELECT COUNT(*) AS cnt FROM tables() WHERE table_name = '%s'", tableName))) { - Assert.assertTrue(rs.next()); - final long actualSize = rs.getLong(1); - Assert.assertEquals(1, actualSize); - } catch (SQLException e) { - throw new RuntimeException(e); - } - }, 5); - } - - /** - * Assert that table has expected size, polling until it does or timeout is - * reached. - */ - protected void assertTableSizeEventually(CharSequence tableName, int expectedSize) throws Exception { - final String sql = String.format("SELECT COUNT(*) AS cnt FROM \"%s\"", tableName); - assertEventually(() -> { - try ( - Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - Assert.assertTrue(rs.next()); - final long actualSize = rs.getLong(1); - Assert.assertEquals(expectedSize, actualSize); - } catch (SQLException e) { - // If the table does not exist yet, we may get an exception - if (e.getMessage().contains("table does not exist")) { - Assert.fail("Table not found: " + tableName); - } - throw new RuntimeException(e); - } - }, 15); - } - - /** - * Drop a table if it exists. - */ - protected void dropTable(String tableName) { - try { - executeSql(String.format("DROP TABLE IF EXISTS '%s'", tableName)); - LOG.info("Dropped table: {}", tableName); - } catch (SQLException e) { - LOG.warn("Failed to drop table {}: {}", tableName, e.getMessage()); - } - } - - /** - * Drop multiple tables. - */ - protected void dropTables(List tableNames) { - for (String tableName : tableNames) { - dropTable(tableName); - } - } - - /** - * Execute SQL and assert no exceptions are thrown. - */ - protected void execute(String sql) throws Exception { - try (Statement statement = getPgConnection().createStatement()) { - statement.execute(sql); - } - } - - /** - * Execute a SELECT query and return results as a list of maps. - * Each map represents a row with column names as keys. - */ - protected List> executeQuery(String sql) throws SQLException { - List> results = new ArrayList<>(); - - try (Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - - ResultSetMetaData metaData = rs.getMetaData(); - int columnCount = metaData.getColumnCount(); - - while (rs.next()) { - Map row = new LinkedHashMap<>(); - for (int i = 1; i <= columnCount; i++) { - String columnName = metaData.getColumnName(i); - Object value = rs.getObject(i); - row.put(columnName, value); - } - results.add(row); - } - } - - return results; - } - - /** - * Execute a DDL or DML statement (CREATE, DROP, INSERT, etc.). - */ - protected void executeSql(String sql) throws SQLException { - try (Statement stmt = getPgConnection().createStatement()) { - stmt.execute(sql); - } - } - - /** - * Generate a unique table name with the given prefix. - * This ensures test isolation when running tests in parallel. - */ - protected String generateTableName(String prefix) { - return prefix + "_" + TABLE_NAME_COUNTER.incrementAndGet(); - } - - /** - * Query table contents with optional ORDER BY clause and return as - * TSV-formatted string. - */ - protected String queryTableAsTsv(String tableName, String orderBy) throws SQLException { - StringBuilder sb = new StringBuilder(); - - String sql = String.format("SELECT * FROM '%s'", tableName); - if (orderBy != null && !orderBy.isEmpty()) { - sql += " ORDER BY " + orderBy; - } - - try (Statement stmt = getPgConnection().createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - - ResultSetMetaData metaData = rs.getMetaData(); - int columnCount = metaData.getColumnCount(); - - // Header row - for (int i = 1; i <= columnCount; i++) { - if (i > 1) { - sb.append('\t'); - } - sb.append(metaData.getColumnName(i)); - } - sb.append('\n'); - - // Data rows - while (rs.next()) { - for (int i = 1; i <= columnCount; i++) { - if (i > 1) { - sb.append('\t'); - } - Object value = rs.getObject(i); - sb.append(value == null ? "" : value.toString()); - } - sb.append('\n'); - } - } - - return sb.toString(); - } - - /** - * Check if table exists (non-blocking, no polling). - */ - protected boolean tableExists(String tableName) throws SQLException { - List> result = executeQuery( - String.format("SELECT table_name FROM tables() WHERE table_name = '%s'", tableName)); - return !result.isEmpty(); - } - - /** - * Track a table for cleanup after the test. - * If the table already exists, we block until it is dropped. - * - * @param tableName the name of the table to track - */ - protected void useTable(String tableName) throws Exception { - TestUtils.assertEventually(() -> { - try { - if (!tableExists(tableName)) { - return; - } - Thread.sleep(100); - dropTable(tableName); - Assert.fail("Table " + tableName + " already exists. Dropping it."); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting for table to be dropped: " + tableName, e); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - - static { - DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() - .parseCaseInsensitive() - .append(DateTimeFormatter.ISO_LOCAL_DATE) - .appendLiteral('T') - .appendValue(HOUR_OF_DAY, 2) - .appendLiteral(':') - .appendValue(MINUTE_OF_HOUR, 2) - .appendLiteral(':') - .appendValue(SECOND_OF_MINUTE, 2) - .appendFraction(NANO_OF_SECOND, 9, 9, true) - .appendLiteral('Z') - .toFormatter(); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java deleted file mode 100644 index 323b6c8..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/AbstractLineSenderTest.java +++ /dev/null @@ -1,127 +0,0 @@ -package io.questdb.client.test.cutlass.line; - -import io.questdb.client.cutlass.line.AbstractLineSender; -import io.questdb.client.test.AbstractQdbTest; - -import java.lang.reflect.Array; - -public class AbstractLineSenderTest extends AbstractQdbTest { - private static final int DEFAULT_ILP_TCP_PORT = 9009; - private static final int DEFAULT_ILP_UDP_PORT = 9009; - - public static T createDoubleArray(int... shape) { - int[] indices = new int[shape.length]; - return buildNestedArray(ArrayDataType.DOUBLE, shape, 0, indices); - } - - @SuppressWarnings("unchecked") - private static T buildNestedArray(ArrayDataType dataType, int[] shape, int currentDim, int[] indices) { - if (currentDim == shape.length - 1) { - Object arr = dataType.createArray(shape[currentDim]); - for (int i = 0; i < Array.getLength(arr); i++) { - indices[currentDim] = i; - dataType.setElement(arr, i, indices); - } - return (T) arr; - } else { - Class componentType = dataType.getComponentType(shape.length - currentDim - 1); - Object arr = Array.newInstance(componentType, shape[currentDim]); - for (int i = 0; i < shape[currentDim]; i++) { - indices[currentDim] = i; - Object subArr = buildNestedArray(dataType, shape, currentDim + 1, indices); - Array.set(arr, i, subArr); - } - return (T) arr; - } - } - - /** - * Get ILP TCP port. - */ - protected static int getIlpTcpPort() { - return getConfigInt("QUESTDB_ILP_TCP_PORT", "questdb.ilp.tcp.port", DEFAULT_ILP_TCP_PORT); - } - - /** - * Get ILP UDP port. - */ - protected static int getIlpUdpPort() { - return getConfigInt("QUESTDB_ILP_UDP_PORT", "questdb.ilp.udp.port", DEFAULT_ILP_UDP_PORT); - } - - /** - * Send data using the sender and assert the expected row count. - * This method flushes the sender, waits for UDP to settle, and polls for the expected row count. - *

      - * UDP is fire-and-forget, so we need extra delay to ensure the server has processed the data. - * - * @param sender the sender to flush - * @param tableName the table to check - * @param expectedRowCount the expected number of rows - */ - protected void flushAndAssertRowCount(AbstractLineSender sender, String tableName, int expectedRowCount) throws Exception { - sender.flush(); - assertTableSizeEventually(tableName, expectedRowCount); - } - - private enum ArrayDataType { - DOUBLE(double.class) { - @Override - public Object createArray(int length) { - return new double[length]; - } - - @Override - public void setElement(Object array, int index, int[] indices) { - double[] arr = (double[]) array; - double product = 1.0; - for (int idx : indices) { - product *= (idx + 1); - } - arr[index] = product; - } - }, - LONG(long.class) { - @Override - public Object createArray(int length) { - return new long[length]; - } - - @Override - public void setElement(Object array, int index, int[] indices) { - long[] arr = (long[]) array; - long product = 1L; - for (int idx : indices) { - product *= (idx + 1); - } - arr[index] = product; - } - }; - - private final Class baseType; - private final Class[] componentTypes = new Class[17]; // 支持最多16维 - - ArrayDataType(Class baseType) { - this.baseType = baseType; - initComponentTypes(); - } - - public abstract Object createArray(int length); - - public Class getComponentType(int dimsRemaining) { - if (dimsRemaining < 0 || dimsRemaining > 16) { - throw new RuntimeException("Array dimension too large"); - } - return componentTypes[dimsRemaining]; - } - - public abstract void setElement(Object array, int index, int[] indices); - - private void initComponentTypes() { - componentTypes[0] = baseType; - for (int dim = 1; dim <= 16; dim++) { - componentTypes[dim] = Array.newInstance(componentTypes[dim - 1], 0).getClass(); - } - } - } -} From ca4763399f4ac9f33ff8d88ba2856b6453e8bf68 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 13 Mar 2026 08:21:26 +0100 Subject: [PATCH 188/230] Add test verifying global auto-flush accumulation across tables The test covers the scenario where rows are written to multiple tables interleaved (t1, t2, t1, t2, ...) to confirm that auto-flush counts rows globally rather than per-table. A bug was reported where auto-flush seemed to trigger on each table switch instead of accumulating the configured total number of rows. The new test (testAutoFlushAccumulatesRowsAcrossAllTables) uses autoFlushRows=5 with bytes and interval checks disabled, writes 4 interleaved rows across two tables, and asserts: - No flush happens on any of the 4 rows (including table switches) - pendingRowCount reflects the total across all tables - The 5th row triggers the flush by hitting the global row threshold The test confirms the code is correct: QwpWebSocketSender accumulates rows globally via pendingRowCount and shouldAutoFlush() checks that counter against autoFlushRows, with no flush logic in table(). Co-Authored-By: Claude Sonnet 4.6 --- .../client/QwpWebSocketSenderStateTest.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index 60a07e0..17ae7ac 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -41,10 +41,59 @@ *

    • {@code reset()} discards all pending state, not just the current table buffer.
      • *
    • Cached timestamp column references are invalidated during flush operations, * preventing stale writes through freed {@code ColumnBuffer} instances.
      • + *
    • Auto-flush accumulates rows globally across all tables rather than flushing + * per-table on each table switch.
      • *
    */ public class QwpWebSocketSenderStateTest extends AbstractTest { + @Test + public void testAutoFlushAccumulatesRowsAcrossAllTables() throws Exception { + assertMemoryLeak(() -> { + // autoFlushRows=5; bytes and interval are disabled to isolate the row-count check. + // The test verifies that switching tables does NOT trigger a flush — flush fires + // only when the TOTAL pending-row count reaches the configured threshold. + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 5, 0, 0L, 1 + ); + try { + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Write 4 rows interleaved between t1 and t2. + // None of these should trigger auto-flush (4 < 5 = autoFlushRows). + sender.table("t1").longColumn("x", 1).at(1, ChronoUnit.MICROS); + sender.table("t2").longColumn("y", 1).at(1, ChronoUnit.MICROS); + sender.table("t1").longColumn("x", 2).at(2, ChronoUnit.MICROS); + sender.table("t2").longColumn("y", 2).at(2, ChronoUnit.MICROS); + + // All 4 rows must still be buffered — switching tables must not flush. + QwpTableBuffer t1 = sender.getTableBuffer("t1"); + QwpTableBuffer t2 = sender.getTableBuffer("t2"); + Assert.assertEquals("t1 should have 2 buffered rows (no premature flush)", + 2, t1.getRowCount()); + Assert.assertEquals("t2 should have 2 buffered rows (no premature flush)", + 2, t2.getRowCount()); + Assert.assertEquals("pendingRowCount must reflect all 4 rows across both tables", + 4, sender.getPendingRowCount()); + + // The 5th row hits the global threshold and triggers auto-flush. + // The flush fails because client is null, confirming that flush + // was triggered by the row-count threshold, not by the table switch. + boolean flushTriggered = false; + try { + sender.table("t1").longColumn("x", 3).at(3, ChronoUnit.MICROS); + } catch (Exception expected) { + flushTriggered = true; + } + Assert.assertTrue("auto-flush must be triggered on the 5th row", flushTriggered); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + @Test public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { assertMemoryLeak(() -> { From 999464c30a78680d1fea9e5fb4f4eec83921297e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 13 Mar 2026 11:10:23 +0100 Subject: [PATCH 189/230] Allow timestamp-only rows in QWP UDP sender --- .../cutlass/qwp/client/QwpUdpSender.java | 10 ------ .../cutlass/qwp/client/QwpUdpSenderTest.java | 35 +++++++++++-------- .../client/QwpWebSocketSenderStateTest.java | 28 +++++++++++++++ 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 610924a..4dcf209 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -719,9 +719,6 @@ private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer column, Object val } private void atMicros(long timestampMicros) { - if (inProgressRowValueCount == 0) { - throw new LineSenderException("no columns were provided"); - } try { stageDesignatedTimestampValue(timestampMicros, false); commitCurrentRow(); @@ -732,9 +729,6 @@ private void atMicros(long timestampMicros) { } private void atNanos(long timestampNanos) { - if (inProgressRowValueCount == 0) { - throw new LineSenderException("no columns were provided"); - } try { stageDesignatedTimestampValue(timestampNanos, true); commitCurrentRow(); @@ -817,10 +811,6 @@ private void clearTransientRowState() { } private void commitCurrentRow() { - if (inProgressRowValueCount == 0) { - throw new LineSenderException("no columns were provided"); - } - long estimate = 0; long committedEstimateBeforeRow = 0; int targetRows = currentTableBuffer.getRowCount() + 1; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 2bd95fb..b436ed0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -1580,6 +1580,27 @@ public void testSymbolPrefixFlushKeepsSingleRetainedDictionaryEntry() throws Exc }); } + @Test + public void testTimestampOnlyRows() throws Exception { + assertMemoryLeak(() -> { + CapturingNetworkFacade nf = new CapturingNetworkFacade(); + try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1)) { + // at() with no other columns: designated timestamp is staged + sender.table("t").at(1_000L, ChronoUnit.MICROS); + // atNow() with no other columns: server assigns the timestamp + sender.table("t").atNow(); + sender.flush(); + } + + List rows = decodeRows(nf.packets); + Assert.assertEquals("expected 2 timestamp-only rows", 2, rows.size()); + assertRowsEqual(Arrays.asList( + decodedRow("t", "", 1_000L), + decodedRow("t", "", null) + ), rows); + }); + } + @Test public void testUnboundedSenderOmittedNullableAndNonNullableColumnsPreservesRows() throws Exception { assertMemoryLeak(() -> { @@ -1712,20 +1733,6 @@ public void testUtf8StringAndSymbolStagingSupportsCancelAndPacketSizing() throws }); } - @Test - public void testZeroColumnRowsThrow() throws Exception { - assertMemoryLeak(() -> { - CapturingNetworkFacade nf = new CapturingNetworkFacade(); - try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { - sender.table("t"); - - assertThrowsContains("no columns were provided", sender::atNow); - assertThrowsContains("no columns were provided", () -> sender.at(1, ChronoUnit.MICROS)); - assertThrowsContains("no columns were provided", () -> sender.at(1, ChronoUnit.NANOS)); - } - }); - } - private static void assertEstimateAtLeastActual(List rows) throws Exception { CapturingNetworkFacade nf = new CapturingNetworkFacade(); try (QwpUdpSender sender = new QwpUdpSender(nf, 0, 0, 9000, 1, 1024 * 1024)) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index 17ae7ac..6e89f98 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -218,6 +218,34 @@ public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception }); } + @Test + public void testTimestampOnlyRows() throws Exception { + assertMemoryLeak(() -> { + // autoFlushRows=10_000 prevents auto-flush; bytes and interval disabled + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 10_000, 0, 0L, 1 + ); + try { + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // at(micros) with no other columns + sender.table("t").at(1_000L, ChronoUnit.MICROS); + // atNow() with no other columns + sender.table("t").atNow(); + + QwpTableBuffer tb = sender.getTableBuffer("t"); + Assert.assertEquals( + "at() and atNow() with no other columns must each buffer a row", + 2, tb.getRowCount() + ); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + private static void setField(Object target, String fieldName, Object value) throws Exception { Field f = target.getClass().getDeclaredField(fieldName); f.setAccessible(true); From 6fd5b613610bc5c31c2c02be3f76384fe9013db3 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 13 Mar 2026 13:17:25 +0100 Subject: [PATCH 190/230] Drain all buffered ACKs per I/O loop iteration Previously, tryReceiveAcks() called tryReceiveFrame() exactly once per I/O loop iteration. In the DRAINING state (all batches sent, ACKs still pending), the I/O thread sleeps 10 ms after each call. This meant each in-flight frame added ~10 ms to flush latency: N in-flight frames incurred ~N x 10 ms overhead, even when all ACKs were already sitting in the TCP receive buffer. Fix by looping tryReceiveFrame() until it returns false, draining all buffered ACKs in a single pass. The 10 ms sleep is now reached only when no more data is available, which is the correct condition for avoiding a busy loop. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/cutlass/qwp/client/WebSocketSendQueue.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index b155bab..d5c39b4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -555,7 +555,9 @@ private void sendBatch(MicrobatchBuffer batch) { */ private void tryReceiveAcks() { try { - client.tryReceiveFrame(responseHandler); + while (client.tryReceiveFrame(responseHandler)) { + // Drain all buffered ACKs before returning to the I/O loop. + } } catch (Exception e) { if (running) { LOG.error("Error receiving response: {}", e.getMessage()); From c33eea94294cd2702157c042da1594077b64ddb5 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 13 Mar 2026 13:33:59 +0100 Subject: [PATCH 191/230] fast defaults for qwp client --- .../main/java/io/questdb/client/Sender.java | 77 +++++++-------- .../qwp/client/QwpWebSocketSender.java | 20 +++- .../line/tcp/v4/QwpAllocationTestClient.java | 3 +- .../line/tcp/v4/StacBenchmarkClient.java | 3 +- .../qwp/client/LineSenderBuilderUdpTest.java | 14 +-- .../LineSenderBuilderWebSocketTest.java | 95 ++++++++----------- 6 files changed, 97 insertions(+), 115 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index af1dbde..f63b92c 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -536,7 +536,7 @@ final class LineSenderBuilder { private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024; private static final int DEFAULT_HTTP_PORT = 9000; private static final int DEFAULT_HTTP_TIMEOUT = 30_000; - private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8; + private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 128; private static final int DEFAULT_MAXIMUM_BUFFER_CAPACITY = 100 * 1024 * 1024; private static final int DEFAULT_MAX_BACKOFF_MILLIS = 1_000; private static final int DEFAULT_MAX_DATAGRAM_SIZE = 1400; @@ -546,9 +546,9 @@ final class LineSenderBuilder { private static final int DEFAULT_TCP_PORT = 9009; private static final int DEFAULT_UDP_PORT = 9007; private static final int DEFAULT_WEBSOCKET_PORT = 9000; - private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB + private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 128 * 1024; // 128KB private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms - private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500; + private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 1_000; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // The PARAMETER_NOT_SET_EXPLICITLY constant is used to detect if a parameter was set explicitly in configuration parameters // where it matters. This is needed to detect invalid combinations of parameters. Why? @@ -561,7 +561,6 @@ final class LineSenderBuilder { private static final int PROTOCOL_WEBSOCKET = 2; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); - private boolean asyncMode = false; private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY; private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; private int autoFlushRows = PARAMETER_NOT_SET_EXPLICITLY; @@ -701,20 +700,18 @@ public AdvancedTlsSettings advancedTls() { } /** - * Enable asynchronous mode for WebSocket transport. + * @deprecated Async mode is now derived from {@link #inFlightWindowSize(int)}. + * Window size 1 implies synchronous mode, greater than 1 implies asynchronous mode. + * The default window size is 128 (asynchronous). Call {@code inFlightWindowSize(1)} + * for synchronous behavior. *
    - * In async mode, rows are batched and sent asynchronously with flow control. - * This provides higher throughput at the cost of more complex error handling. - *
    - * This is only used when communicating over WebSocket transport. - *
    - * Default is synchronous mode (false). + * This method is a no-op and will be removed in a future release. * - * @param enabled whether to enable async mode + * @param enabled ignored * @return this instance for method chaining */ + @Deprecated public LineSenderBuilder asyncMode(boolean enabled) { - this.asyncMode = enabled; return this; } @@ -723,7 +720,7 @@ public LineSenderBuilder asyncMode(boolean enabled) { *
    * This is only used when communicating over WebSocket transport. *
    - * Default value is 1MB. + * Default value is 128KB. * * @param bytes maximum bytes per batch * @return this instance for method chaining @@ -888,28 +885,16 @@ public Sender build() { String wsAuthHeader = buildWebSocketAuthHeader(); - if (asyncMode) { - return QwpWebSocketSender.connectAsync( - hosts.getQuick(0), - ports.getQuick(0), - tlsEnabled, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - actualInFlightWindowSize, - wsAuthHeader - ); - } else { - return QwpWebSocketSender.connect( - hosts.getQuick(0), - ports.getQuick(0), - tlsEnabled, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - wsAuthHeader - ); - } + return QwpWebSocketSender.connectAsync( + hosts.getQuick(0), + ports.getQuick(0), + tlsEnabled, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + actualInFlightWindowSize, + wsAuthHeader + ); } if (protocol == PROTOCOL_UDP) { @@ -1177,9 +1162,12 @@ public LineSenderBuilder httpUsernamePassword(String username, String password) /** * Set the maximum number of batches that can be in-flight awaiting server acknowledgment. *
    - * This is only used when communicating over WebSocket transport with async mode enabled. + * This is only used when communicating over WebSocket transport. + *
    + * A value of 1 means synchronous mode: each batch waits for an ACK before sending the next one. + * A value greater than 1 enables asynchronous mode with pipelined sends and a background I/O thread. *
    - * Default value is 8. + * Default value is 128 (asynchronous). * * @param size maximum number of in-flight batches * @return this instance for method chaining @@ -1774,6 +1762,13 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { int protocolVersion = parseIntValue(sink, "protocol_version"); protocolVersion(protocolVersion); } + } else if (Chars.equals("in_flight_window", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("in_flight_window is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "in_flight_window"); + int windowSize = parseIntValue(sink, "in_flight_window"); + inFlightWindowSize(windowSize); } else if (Chars.equals("max_datagram_size", sink)) { pos = getValue(configurationString, pos, sink, "max_datagram_size"); int mds = parseIntValue(sink, "max_datagram_size"); @@ -1929,9 +1924,6 @@ private void validateParameters() { if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol version is not supported for UDP transport"); } - if (asyncMode) { - throw new LineSenderException("async mode is not supported for UDP transport"); - } if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("in-flight window size is not supported for UDP transport"); } @@ -1957,9 +1949,6 @@ private void validateParameters() { if (httpToken != null && (username != null || password != null)) { throw new LineSenderException("cannot use both token and username/password authentication"); } - if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { - throw new LineSenderException("in-flight window size requires async mode"); - } if (httpPath != null) { throw new LineSenderException("HTTP path is not supported for WebSocket protocol"); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index ef900a2..0c19b45 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -88,10 +88,10 @@ */ public class QwpWebSocketSender implements Sender { - public static final int DEFAULT_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB + public static final int DEFAULT_AUTO_FLUSH_BYTES = 128 * 1024; // 128KB public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms - public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; - public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 + public static final int DEFAULT_AUTO_FLUSH_ROWS = 1_000; + public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 128; private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MAX_NAME_LENGTH = 127; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB @@ -251,7 +251,12 @@ public static QwpWebSocketSender connect( 1, // window=1 for sync behavior authorizationHeader ); - sender.ensureConnected(); + try { + sender.ensureConnected(); + } catch (Throwable t) { + sender.close(); + throw t; + } return sender; } @@ -318,7 +323,12 @@ public static QwpWebSocketSender connectAsync( QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, authorizationHeader ); - sender.ensureConnected(); + try { + sender.ensureConnected(); + } catch (Throwable t) { + sender.close(); + throw t; + } return sender; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index b3f5a98..bbf3c19 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -207,8 +207,7 @@ private static Sender createSender( case PROTOCOL_QWP_WEBSOCKET: Sender.LineSenderBuilder wsBuilder = Sender.builder(Sender.Transport.WEBSOCKET) .address(host) - .port(port) - .asyncMode(true); + .port(port); if (batchSize > 0) wsBuilder.autoFlushRows(batchSize); if (flushBytes > 0) wsBuilder.autoFlushBytes(flushBytes); if (flushIntervalMs > 0) wsBuilder.autoFlushIntervalMillis((int) flushIntervalMs); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index 1eb044d..b7707f1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -189,8 +189,7 @@ private static Sender createSender(String protocol, String host, int port, case PROTOCOL_QWP_WEBSOCKET: Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) .address(host) - .port(port) - .asyncMode(true); + .port(port); if (batchSize > 0) b.autoFlushRows(batchSize); if (flushBytes > 0) b.autoFlushBytes(flushBytes); if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java index 632a549..663d61e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java @@ -86,15 +86,6 @@ public void testUdp_authNotSupported() { "not supported for UDP"); } - @Test - public void testUdp_asyncModeNotSupported() { - assertThrowsAny( - Sender.builder(Sender.Transport.UDP) - .address("localhost") - .asyncMode(true), - "not supported for UDP"); - } - @Test public void testUdp_autoFlushBytesNotSupported() { assertThrowsAny( @@ -278,6 +269,11 @@ public void testUdp_tlsEnabled_throws() { "TLS is not supported for UDP"); } + @Test + public void testUdpScheme_inFlightWindow_fails() { + assertBadConfig("udp::addr=localhost:9007;in_flight_window=64;", "only supported for WebSocket"); + } + @Test public void testUdp_tokenNotSupported() { assertBadConfig("udp::addr=localhost:9007;token=foo;", "token is not supported for UDP"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 787c70c..c0c38c3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -76,36 +76,10 @@ public void testAddressWithoutPort_usesDefaultPort9000() { Assert.assertNotNull(builder); } - @Test - public void testAsyncModeCanBeSetMultipleTimes() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .asyncMode(false); - Assert.assertNotNull(builder); - } - - @Test - public void testAsyncModeDisabled() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false); - Assert.assertNotNull(builder); - } - - @Test - public void testAsyncModeEnabled() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true); - Assert.assertNotNull(builder); - } - @Test public void testAsyncModeWithAllOptions() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .asyncMode(true) .autoFlushRows(500) .autoFlushBytes(512 * 1024) .autoFlushIntervalMillis(50) @@ -322,7 +296,6 @@ public void testEnableAuth_notSupported() { public void testFullAsyncConfiguration() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .asyncMode(true) .autoFlushRows(1000) .autoFlushBytes(1024 * 1024) .autoFlushIntervalMillis(100) @@ -336,7 +309,6 @@ public void testFullAsyncConfigurationWithTls() { .address(LOCALHOST) .enableTls() .advancedTls().disableCertificateValidation() - .asyncMode(true) .autoFlushRows(1000) .autoFlushBytes(1024 * 1024) .inFlightWindowSize(16); @@ -374,7 +346,6 @@ public void testInFlightWindowSizeDoubleSet_fails() { assertThrows("already configured", () -> Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .asyncMode(true) .inFlightWindowSize(8) .inFlightWindowSize(16)); } @@ -384,37 +355,33 @@ public void testInFlightWindowSizeNegative_fails() { assertThrows("must be positive", () -> Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .asyncMode(true) .inFlightWindowSize(-1)); } + @Test + public void testInFlightWindowSizeOne_syncMode() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(1); + Assert.assertNotNull(builder); + } + @Test public void testInFlightWindowSizeZero_fails() { assertThrows("must be positive", () -> Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .asyncMode(true) .inFlightWindowSize(0)); } @Test - public void testInFlightWindowSize_withAsyncMode() { + public void testInFlightWindowSize_customValue() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .asyncMode(true) .inFlightWindowSize(16); Assert.assertNotNull(builder); } - @Test - public void testInFlightWindowSize_withoutAsyncMode_fails() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .inFlightWindowSize(16), - "requires async mode"); - } - @Test public void testInvalidPort_fails() { assertThrows("invalid port", @@ -545,17 +512,8 @@ public void testSyncModeAutoFlushDefaults() throws Exception { } @Test - public void testSyncModeDoesNotAllowInFlightWindowSize() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false) - .inFlightWindowSize(16), - "requires async mode"); - } - - @Test - public void testSyncModeIsDefault() { + public void testDefaultIsAsync() { + // Default in-flight window size is 128 (async) Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST); Assert.assertNotNull(builder); @@ -613,6 +571,37 @@ public void testUsernamePassword_accepted() { Assert.assertNotNull(builder); } + @Test + public void testWsConfigString_inFlightWindow() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";in_flight_window=64;", "connect", "Failed"); + }); + } + + @Test + public void testWsConfigString_inFlightWindowDoubleSet_fails() { + assertBadConfig("ws::addr=localhost:9000;in_flight_window=64;in_flight_window=128;", "already configured"); + } + + @Test + public void testWsConfigString_inFlightWindowInvalid_fails() { + assertBadConfig("ws::addr=localhost:9000;in_flight_window=0;", "must be positive"); + } + + @Test + public void testWsConfigString_inFlightWindowNotSupportedForHttp_fails() { + assertBadConfig("http::addr=localhost:9000;in_flight_window=64;", "only supported for WebSocket"); + } + + @Test + public void testWsConfigString_inFlightWindowSync() throws Exception { + assertMemoryLeak(() -> { + int port = findUnusedPort(); + assertBadConfig("ws::addr=localhost:" + port + ";in_flight_window=1;", "connect", "Failed"); + }); + } + @Test public void testWsConfigString() throws Exception { assertMemoryLeak(() -> { From a1850ca3e389ff892b3f5bc10cdb202c1bb0147f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 13 Mar 2026 13:54:29 +0100 Subject: [PATCH 192/230] Downgrade java-questdb-client to Java 11 Replace Java 14+ enhanced switch expressions (arrow syntax, yield, multi-case labels) with traditional switch statements. Convert Java 16 pattern variables in instanceof to explicit casts, and replace the AckFrameHandler record with a static inner class. Update pom.xml to target Java 11: javac.target, javadoc source versions, and the compiler profile activation threshold. Co-Authored-By: Claude Opus 4.6 --- core/pom.xml | 18 +-- .../main/java/io/questdb/client/Sender.java | 65 +++++--- .../http/client/WebSocketClientFactory.java | 17 +- .../cutlass/qwp/client/MicrobatchBuffer.java | 19 ++- .../cutlass/qwp/client/QwpColumnWriter.java | 7 +- .../cutlass/qwp/client/QwpUdpSender.java | 150 +++++++++++------- .../qwp/client/QwpWebSocketSender.java | 37 +++-- .../cutlass/qwp/client/WebSocketResponse.java | 25 +-- .../cutlass/qwp/protocol/QwpConstants.java | 137 +++++++++++----- .../qwp/protocol/QwpGorillaEncoder.java | 39 +++-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 125 ++++++++++----- .../qwp/websocket/WebSocketCloseCode.java | 52 +++--- .../qwp/websocket/TestWebSocketServer.java | 11 +- 13 files changed, 472 insertions(+), 230 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index a117725..3299370 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -32,7 +32,7 @@ false target none - 17 + 11 -ea -Dfile.encoding=UTF-8 -XX:+UseParallelGC None %regex[.*[^o].class] @@ -198,7 +198,7 @@ none - 17 + 11 false ${compilerArg1} @@ -296,7 +296,7 @@ none - 17 + 11 false ${compilerArg1} @@ -384,20 +384,20 @@ - java17+ + java11+ - 17 - 17 + 11 + 11 questdb --add-exports java.base/jdk.internal.math=io.questdb.client - nothing-to-exclude-dummy-value-include-all-java17plus - nothing-to-exclude-dummy-value-include-all-java17plus + nothing-to-exclude-dummy-value-include-all-java11plus + nothing-to-exclude-dummy-value-include-all-java11plus ${javac.target} ${javac.target} - [17,) + [11,) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index f63b92c..00b86af 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -150,12 +150,23 @@ static LineSenderBuilder builder(CharSequence configurationString) { * @return Builder object to create a new Sender instance. */ static LineSenderBuilder builder(Transport transport) { - int protocol = switch (transport) { - case HTTP -> LineSenderBuilder.PROTOCOL_HTTP; - case TCP -> LineSenderBuilder.PROTOCOL_TCP; - case UDP -> LineSenderBuilder.PROTOCOL_UDP; - case WEBSOCKET -> LineSenderBuilder.PROTOCOL_WEBSOCKET; - }; + int protocol; + switch (transport) { + case HTTP: + protocol = LineSenderBuilder.PROTOCOL_HTTP; + break; + case TCP: + protocol = LineSenderBuilder.PROTOCOL_TCP; + break; + case UDP: + protocol = LineSenderBuilder.PROTOCOL_UDP; + break; + case WEBSOCKET: + protocol = LineSenderBuilder.PROTOCOL_WEBSOCKET; + break; + default: + throw new IllegalArgumentException("unknown transport: " + transport); + } return new LineSenderBuilder(protocol); } @@ -927,13 +938,19 @@ public Sender build() { channel = tlsChannel; } try { - sender = switch (protocolVersion) { - case PROTOCOL_VERSION_V1 -> new LineTcpSenderV1(channel, bufferCapacity, maxNameLength); - case PROTOCOL_VERSION_V2 -> new LineTcpSenderV2(channel, bufferCapacity, maxNameLength); - case PROTOCOL_VERSION_V3 -> new LineTcpSenderV3(channel, bufferCapacity, maxNameLength); - default -> - throw new LineSenderException("unknown protocol version [version=").put(protocolVersion).put("]"); - }; + switch (protocolVersion) { + case PROTOCOL_VERSION_V1: + sender = new LineTcpSenderV1(channel, bufferCapacity, maxNameLength); + break; + case PROTOCOL_VERSION_V2: + sender = new LineTcpSenderV2(channel, bufferCapacity, maxNameLength); + break; + case PROTOCOL_VERSION_V3: + sender = new LineTcpSenderV3(channel, bufferCapacity, maxNameLength); + break; + default: + throw new LineSenderException("unknown protocol version [version=").put(protocolVersion).put("]"); + } } catch (Throwable t) { channel.close(); throw rethrow(t); @@ -1541,14 +1558,24 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { throw new LineSenderException("invalid configuration string: ").put(sink); } if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + String protocolName; + switch (protocol) { + case PROTOCOL_HTTP: + protocolName = "http"; + break; + case PROTOCOL_UDP: + protocolName = "udp"; + break; + case PROTOCOL_WEBSOCKET: + protocolName = "websocket"; + break; + default: + protocolName = "tcp"; + break; + } throw new LineSenderException("protocol was already configured ") .put("[protocol=") - .put(switch (protocol) { - case PROTOCOL_HTTP -> "http"; - case PROTOCOL_UDP -> "udp"; - case PROTOCOL_WEBSOCKET -> "websocket"; - default -> "tcp"; - }).put("]"); + .put(protocolName).put("]"); } if (Chars.equals("http", sink)) { if (tlsEnabled) { diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java index c6b36d2..4972176 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java @@ -86,12 +86,17 @@ public static WebSocketClient newInsecureTlsInstance() { * @return a new platform-specific WebSocket client */ public static WebSocketClient newInstance(HttpClientConfiguration configuration, SocketFactory socketFactory) { - return switch (Os.type) { - case Os.LINUX -> new WebSocketClientLinux(configuration, socketFactory); - case Os.DARWIN, Os.FREEBSD -> new WebSocketClientOsx(configuration, socketFactory); - case Os.WINDOWS -> new WebSocketClientWindows(configuration, socketFactory); - default -> throw new UnsupportedOperationException("Unsupported platform: " + Os.type); - }; + switch (Os.type) { + case Os.LINUX: + return new WebSocketClientLinux(configuration, socketFactory); + case Os.DARWIN: + case Os.FREEBSD: + return new WebSocketClientOsx(configuration, socketFactory); + case Os.WINDOWS: + return new WebSocketClientWindows(configuration, socketFactory); + default: + throw new UnsupportedOperationException("Unsupported platform: " + Os.type); + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 4f5bfbf..38d29a3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -118,13 +118,18 @@ public MicrobatchBuffer(int initialCapacity) { * Returns a human-readable name for the given state. */ public static String stateName(int state) { - return switch (state) { - case STATE_FILLING -> "FILLING"; - case STATE_SEALED -> "SEALED"; - case STATE_SENDING -> "SENDING"; - case STATE_RECYCLED -> "RECYCLED"; - default -> "UNKNOWN(" + state + ")"; - }; + switch (state) { + case STATE_FILLING: + return "FILLING"; + case STATE_SEALED: + return "SEALED"; + case STATE_SENDING: + return "SENDING"; + case STATE_RECYCLED: + return "RECYCLED"; + default: + return "UNKNOWN(" + state + ")"; + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index 81c95e0..b3a5896 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -73,10 +73,13 @@ private void encodeColumn( case TYPE_CHAR: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); break; - case TYPE_INT, TYPE_FLOAT: + case TYPE_INT: + case TYPE_FLOAT: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; - case TYPE_LONG, TYPE_DATE, TYPE_DOUBLE: + case TYPE_LONG: + case TYPE_DATE: + case TYPE_DOUBLE: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_TIMESTAMP: diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 4dcf209..fcb2d78 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -554,17 +554,31 @@ private static long estimateInProgressColumnPayload(InProgressColumnState state) return 0; } - return switch (col.getType()) { - case TYPE_BOOLEAN -> packedBytes(valueCountAfter) - packedBytes(valueCountBefore); - case TYPE_DECIMAL64 -> 8; - case TYPE_DECIMAL128 -> 16; - case TYPE_DECIMAL256 -> 32; - case TYPE_DOUBLE, TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> 8; - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> estimateArrayPayloadBytes(col, state); - case TYPE_STRING, TYPE_VARCHAR -> 4L + (col.getStringDataSize() - state.stringDataSizeBefore); - case TYPE_SYMBOL -> estimateSymbolPayloadDelta(col, state); - default -> throw new LineSenderException("unsupported in-progress column type: " + col.getType()); - }; + switch (col.getType()) { + case TYPE_BOOLEAN: + return packedBytes(valueCountAfter) - packedBytes(valueCountBefore); + case TYPE_DECIMAL64: + return 8; + case TYPE_DECIMAL128: + return 16; + case TYPE_DECIMAL256: + return 32; + case TYPE_DOUBLE: + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + return 8; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + return estimateArrayPayloadBytes(col, state); + case TYPE_STRING: + case TYPE_VARCHAR: + return 4L + (col.getStringDataSize() - state.stringDataSizeBefore); + case TYPE_SYMBOL: + return estimateSymbolPayloadDelta(col, state); + default: + throw new LineSenderException("unsupported in-progress column type: " + col.getType()); + } } private static long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, InProgressColumnState state) { @@ -602,22 +616,44 @@ private static long estimateSymbolPayloadDelta(QwpTableBuffer.ColumnBuffer col, } private static long nonNullablePaddingCost(byte type, int valuesBefore, int missing) { - return switch (type) { - case TYPE_BOOLEAN -> packedBytes(valuesBefore + missing) - packedBytes(valuesBefore); - case TYPE_BYTE -> missing; - case TYPE_SHORT, TYPE_CHAR -> (long) missing * 2; - case TYPE_INT, TYPE_FLOAT -> (long) missing * 4; - case TYPE_LONG, TYPE_DOUBLE, TYPE_DATE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS -> (long) missing * 8; - case TYPE_UUID -> (long) missing * 16; - case TYPE_LONG256 -> (long) missing * 32; - case TYPE_DECIMAL64 -> (long) missing * 8; - case TYPE_DECIMAL128 -> (long) missing * 16; - case TYPE_DECIMAL256 -> (long) missing * 32; - case TYPE_STRING, TYPE_VARCHAR -> (long) missing * 4; - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> (long) missing * 5; - case TYPE_SYMBOL -> throw new IllegalStateException("symbol columns must be nullable"); - default -> 0; - }; + switch (type) { + case TYPE_BOOLEAN: + return packedBytes(valuesBefore + missing) - packedBytes(valuesBefore); + case TYPE_BYTE: + return missing; + case TYPE_SHORT: + case TYPE_CHAR: + return (long) missing * 2; + case TYPE_INT: + case TYPE_FLOAT: + return (long) missing * 4; + case TYPE_LONG: + case TYPE_DOUBLE: + case TYPE_DATE: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + return (long) missing * 8; + case TYPE_UUID: + return (long) missing * 16; + case TYPE_LONG256: + return (long) missing * 32; + case TYPE_DECIMAL64: + return (long) missing * 8; + case TYPE_DECIMAL128: + return (long) missing * 16; + case TYPE_DECIMAL256: + return (long) missing * 32; + case TYPE_STRING: + case TYPE_VARCHAR: + return (long) missing * 4; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + return (long) missing * 5; + case TYPE_SYMBOL: + throw new IllegalStateException("symbol columns must be nullable"); + default: + return 0; + } } private static int packedBytes(int valueCount) { @@ -668,20 +704,20 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, } private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { - if (value instanceof double[] values) { - column.addDoubleArray(values); + if (value instanceof double[]) { + column.addDoubleArray((double[]) value); return; } - if (value instanceof double[][] values) { - column.addDoubleArray(values); + if (value instanceof double[][]) { + column.addDoubleArray((double[][]) value); return; } - if (value instanceof double[][][] values) { - column.addDoubleArray(values); + if (value instanceof double[][][]) { + column.addDoubleArray((double[][][]) value); return; } - if (value instanceof DoubleArray values) { - column.addDoubleArray(values); + if (value instanceof DoubleArray) { + column.addDoubleArray((DoubleArray) value); return; } throw new LineSenderException("unsupported double array type"); @@ -699,20 +735,20 @@ private void appendInProgressColumnState(QwpTableBuffer.ColumnBuffer column) { } private void appendLongArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { - if (value instanceof long[] values) { - column.addLongArray(values); + if (value instanceof long[]) { + column.addLongArray((long[]) value); return; } - if (value instanceof long[][] values) { - column.addLongArray(values); + if (value instanceof long[][]) { + column.addLongArray((long[][]) value); return; } - if (value instanceof long[][][] values) { - column.addLongArray(values); + if (value instanceof long[][][]) { + column.addLongArray((long[][][]) value); return; } - if (value instanceof LongArray values) { - column.addLongArray(values); + if (value instanceof LongArray) { + column.addLongArray((LongArray) value); return; } throw new LineSenderException("unsupported long array type"); @@ -1243,16 +1279,24 @@ private void stageTimestampColumnValue(CharSequence name, byte type, long value) } private long toMicros(long value, ChronoUnit unit) { - return switch (unit) { - case NANOS -> value / 1000L; - case MICROS -> value; - case MILLIS -> value * 1000L; - case SECONDS -> value * 1_000_000L; - case MINUTES -> value * 60_000_000L; - case HOURS -> value * 3_600_000_000L; - case DAYS -> value * 86_400_000_000L; - default -> throw new LineSenderException("Unsupported time unit: " + unit); - }; + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 0c19b45..1387af5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1350,16 +1350,24 @@ private boolean shouldAutoFlush() { } private long toMicros(long value, ChronoUnit unit) { - return switch (unit) { - case NANOS -> value / 1000L; - case MICROS -> value; - case MILLIS -> value * 1000L; - case SECONDS -> value * 1_000_000L; - case MINUTES -> value * 60_000_000L; - case HOURS -> value * 3_600_000_000L; - case DAYS -> value * 86_400_000_000L; - default -> throw new LineSenderException("Unsupported time unit: " + unit); - }; + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); + } } private void validateTableName(CharSequence name) { @@ -1424,9 +1432,12 @@ private void waitForAck(long expectedSequence) { throw timeout; } - private record AckFrameHandler( - QwpWebSocketSender sender - ) implements WebSocketFrameHandler { + private static class AckFrameHandler implements WebSocketFrameHandler { + private final QwpWebSocketSender sender; + + AckFrameHandler(QwpWebSocketSender sender) { + this.sender = sender; + } @Override public void onBinaryMessage(long payloadPtr, int payloadLen) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index f9f6c01..99b7eba 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -145,15 +145,22 @@ public long getSequence() { * Returns a human-readable status name. */ public String getStatusName() { - return switch (status) { - case STATUS_OK -> "OK"; - case STATUS_PARSE_ERROR -> "PARSE_ERROR"; - case STATUS_SCHEMA_ERROR -> "SCHEMA_ERROR"; - case STATUS_WRITE_ERROR -> "WRITE_ERROR"; - case STATUS_SECURITY_ERROR -> "SECURITY_ERROR"; - case STATUS_INTERNAL_ERROR -> "INTERNAL_ERROR"; - default -> "UNKNOWN(" + (status & 0xFF) + ")"; - }; + switch (status) { + case STATUS_OK: + return "OK"; + case STATUS_PARSE_ERROR: + return "PARSE_ERROR"; + case STATUS_SCHEMA_ERROR: + return "SCHEMA_ERROR"; + case STATUS_WRITE_ERROR: + return "WRITE_ERROR"; + case STATUS_SECURITY_ERROR: + return "SECURITY_ERROR"; + case STATUS_INTERNAL_ERROR: + return "INTERNAL_ERROR"; + default: + return "UNKNOWN(" + (status & 0xFF) + ")"; + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index b605f2e..df55c22 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -270,17 +270,35 @@ private QwpConstants() { */ public static int getFixedTypeSize(byte typeCode) { int code = typeCode & TYPE_MASK; - return switch (code) { - case TYPE_BOOLEAN -> 0; // Special: bit-packed - case TYPE_BYTE -> 1; - case TYPE_SHORT, TYPE_CHAR -> 2; - case TYPE_INT, TYPE_FLOAT -> 4; - case TYPE_LONG, TYPE_DOUBLE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE, TYPE_DECIMAL64 -> 8; - case TYPE_UUID, TYPE_DECIMAL128 -> 16; - case TYPE_LONG256, TYPE_DECIMAL256 -> 32; - case TYPE_GEOHASH -> -1; // Variable width: varint precision + packed values - default -> -1; // Variable width - }; + switch (code) { + case TYPE_BOOLEAN: + return 0; // Special: bit-packed + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_FLOAT: + return 4; + case TYPE_LONG: + case TYPE_DOUBLE: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + case TYPE_DECIMAL64: + return 8; + case TYPE_UUID: + case TYPE_DECIMAL128: + return 16; + case TYPE_LONG256: + case TYPE_DECIMAL256: + return 32; + case TYPE_GEOHASH: + return -1; // Variable width: varint precision + packed values + default: + return -1; // Variable width + } } /** @@ -292,31 +310,78 @@ public static int getFixedTypeSize(byte typeCode) { public static String getTypeName(byte typeCode) { int code = typeCode & TYPE_MASK; boolean nullable = (typeCode & TYPE_NULLABLE_FLAG) != 0; - String name = switch (code) { - case TYPE_BOOLEAN -> "BOOLEAN"; - case TYPE_BYTE -> "BYTE"; - case TYPE_SHORT -> "SHORT"; - case TYPE_CHAR -> "CHAR"; - case TYPE_INT -> "INT"; - case TYPE_LONG -> "LONG"; - case TYPE_FLOAT -> "FLOAT"; - case TYPE_DOUBLE -> "DOUBLE"; - case TYPE_STRING -> "STRING"; - case TYPE_SYMBOL -> "SYMBOL"; - case TYPE_TIMESTAMP -> "TIMESTAMP"; - case TYPE_TIMESTAMP_NANOS -> "TIMESTAMP_NANOS"; - case TYPE_DATE -> "DATE"; - case TYPE_UUID -> "UUID"; - case TYPE_LONG256 -> "LONG256"; - case TYPE_GEOHASH -> "GEOHASH"; - case TYPE_VARCHAR -> "VARCHAR"; - case TYPE_DOUBLE_ARRAY -> "DOUBLE_ARRAY"; - case TYPE_LONG_ARRAY -> "LONG_ARRAY"; - case TYPE_DECIMAL64 -> "DECIMAL64"; - case TYPE_DECIMAL128 -> "DECIMAL128"; - case TYPE_DECIMAL256 -> "DECIMAL256"; - default -> "UNKNOWN(" + code + ")"; - }; + String name; + switch (code) { + case TYPE_BOOLEAN: + name = "BOOLEAN"; + break; + case TYPE_BYTE: + name = "BYTE"; + break; + case TYPE_SHORT: + name = "SHORT"; + break; + case TYPE_CHAR: + name = "CHAR"; + break; + case TYPE_INT: + name = "INT"; + break; + case TYPE_LONG: + name = "LONG"; + break; + case TYPE_FLOAT: + name = "FLOAT"; + break; + case TYPE_DOUBLE: + name = "DOUBLE"; + break; + case TYPE_STRING: + name = "STRING"; + break; + case TYPE_SYMBOL: + name = "SYMBOL"; + break; + case TYPE_TIMESTAMP: + name = "TIMESTAMP"; + break; + case TYPE_TIMESTAMP_NANOS: + name = "TIMESTAMP_NANOS"; + break; + case TYPE_DATE: + name = "DATE"; + break; + case TYPE_UUID: + name = "UUID"; + break; + case TYPE_LONG256: + name = "LONG256"; + break; + case TYPE_GEOHASH: + name = "GEOHASH"; + break; + case TYPE_VARCHAR: + name = "VARCHAR"; + break; + case TYPE_DOUBLE_ARRAY: + name = "DOUBLE_ARRAY"; + break; + case TYPE_LONG_ARRAY: + name = "LONG_ARRAY"; + break; + case TYPE_DECIMAL64: + name = "DECIMAL64"; + break; + case TYPE_DECIMAL128: + name = "DECIMAL128"; + break; + case TYPE_DECIMAL256: + name = "DECIMAL256"; + break; + default: + name = "UNKNOWN(" + code + ")"; + break; + } return nullable ? name + "?" : name; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 5d1ed4f..3d8c104 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -148,13 +148,18 @@ public static boolean canUseGorilla(long srcAddress, int count) { */ public static int getBitsRequired(long deltaOfDelta) { int bucket = getBucket(deltaOfDelta); - return switch (bucket) { - case 0 -> 1; - case 1 -> 9; - case 2 -> 12; - case 3 -> 16; - default -> 36; - }; + switch (bucket) { + case 0: + return 1; + case 1: + return 9; + case 2: + return 12; + case 3: + return 16; + default: + return 36; + } } /** @@ -194,23 +199,25 @@ public static int getBucket(long deltaOfDelta) { public void encodeDoD(long deltaOfDelta) { int bucket = getBucket(deltaOfDelta); switch (bucket) { - case 0 -> bitWriter.writeBit(0); - case 1 -> { + case 0: + bitWriter.writeBit(0); + break; + case 1: bitWriter.writeBits(0b01, 2); bitWriter.writeSigned(deltaOfDelta, 7); - } - case 2 -> { + break; + case 2: bitWriter.writeBits(0b011, 3); bitWriter.writeSigned(deltaOfDelta, 9); - } - case 3 -> { + break; + case 3: bitWriter.writeBits(0b0111, 4); bitWriter.writeSigned(deltaOfDelta, 12); - } - default -> { + break; + default: bitWriter.writeBits(0b1111, 4); bitWriter.writeSigned(deltaOfDelta, 32); - } + break; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index be02af5..af94111 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -416,16 +416,34 @@ private void rebuildColumnAccessStructures() { * @see QwpConstants#getFixedTypeSize(byte) for wire-format sizes */ static int elementSizeInBuffer(byte type) { - return switch (type) { - case TYPE_BOOLEAN, TYPE_BYTE -> 1; - case TYPE_SHORT, TYPE_CHAR -> 2; - case TYPE_INT, TYPE_SYMBOL, TYPE_FLOAT -> 4; - case TYPE_GEOHASH, TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, - TYPE_DATE, TYPE_DECIMAL64, TYPE_DOUBLE -> 8; - case TYPE_UUID, TYPE_DECIMAL128 -> 16; - case TYPE_LONG256, TYPE_DECIMAL256 -> 32; - default -> 0; - }; + switch (type) { + case TYPE_BOOLEAN: + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_SYMBOL: + case TYPE_FLOAT: + return 4; + case TYPE_GEOHASH: + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + case TYPE_DECIMAL64: + case TYPE_DOUBLE: + return 8; + case TYPE_UUID: + case TYPE_DECIMAL128: + return 16; + case TYPE_LONG256: + case TYPE_DECIMAL256: + return 32; + default: + return 0; + } } /** @@ -949,42 +967,68 @@ public void addNull() { } else { // For non-nullable columns, store a sentinel/default value switch (type) { - case TYPE_BOOLEAN, TYPE_BYTE -> dataBuffer.putByte((byte) 0); - case TYPE_SHORT, TYPE_CHAR -> dataBuffer.putShort((short) 0); - case TYPE_INT -> dataBuffer.putInt(0); - case TYPE_GEOHASH -> dataBuffer.putLong(-1L); - case TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE -> - dataBuffer.putLong(Long.MIN_VALUE); - case TYPE_FLOAT -> dataBuffer.putFloat(Float.NaN); - case TYPE_DOUBLE -> dataBuffer.putDouble(Double.NaN); - case TYPE_STRING, TYPE_VARCHAR -> stringOffsets.putInt((int) stringData.getAppendOffset()); - case TYPE_SYMBOL -> dataBuffer.putInt(-1); - case TYPE_UUID -> { + case TYPE_BOOLEAN: + case TYPE_BYTE: + dataBuffer.putByte((byte) 0); + break; + case TYPE_SHORT: + case TYPE_CHAR: + dataBuffer.putShort((short) 0); + break; + case TYPE_INT: + dataBuffer.putInt(0); + break; + case TYPE_GEOHASH: + dataBuffer.putLong(-1L); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_FLOAT: + dataBuffer.putFloat(Float.NaN); + break; + case TYPE_DOUBLE: + dataBuffer.putDouble(Double.NaN); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringOffsets.putInt((int) stringData.getAppendOffset()); + break; + case TYPE_SYMBOL: + dataBuffer.putInt(-1); + break; + case TYPE_UUID: dataBuffer.putLong(Long.MIN_VALUE); - } - case TYPE_LONG256 -> { dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_LONG256: dataBuffer.putLong(Long.MIN_VALUE); dataBuffer.putLong(Long.MIN_VALUE); dataBuffer.putLong(Long.MIN_VALUE); - } - case TYPE_DECIMAL64 -> dataBuffer.putLong(Decimals.DECIMAL64_NULL); - case TYPE_DECIMAL128 -> { + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_DECIMAL64: + dataBuffer.putLong(Decimals.DECIMAL64_NULL); + break; + case TYPE_DECIMAL128: dataBuffer.putLong(Decimals.DECIMAL128_HI_NULL); dataBuffer.putLong(Decimals.DECIMAL128_LO_NULL); - } - case TYPE_DECIMAL256 -> { + break; + case TYPE_DECIMAL256: dataBuffer.putLong(Decimals.DECIMAL256_HH_NULL); dataBuffer.putLong(Decimals.DECIMAL256_HL_NULL); dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); - } - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> { + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: ensureArrayCapacity(1, 0); arrayDims[valueCount] = 1; arrayShapes[arrayShapeOffset++] = 0; - } + break; } valueCount++; } @@ -1314,11 +1358,20 @@ public void retainTailRow( } switch (type) { - case TYPE_STRING, TYPE_VARCHAR -> retainStringValue(valueCountBefore); - case TYPE_SYMBOL -> retainSymbolValue(valueCountBefore); - case TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY -> - retainArrayValue(valueCountBefore, arrayShapeOffsetBefore, arrayDataOffsetBefore); - default -> retainFixedWidthValue(valueCountBefore); + case TYPE_STRING: + case TYPE_VARCHAR: + retainStringValue(valueCountBefore); + break; + case TYPE_SYMBOL: + retainSymbolValue(valueCountBefore); + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + retainArrayValue(valueCountBefore, arrayShapeOffsetBefore, arrayDataOffsetBefore); + break; + default: + retainFixedWidthValue(valueCountBefore); + break; } size = 1; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 6ee2071..0e70e0c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -107,29 +107,41 @@ private WebSocketCloseCode() { * @return the description */ public static String describe(int code) { - return switch (code) { - case NORMAL_CLOSURE -> "Normal Closure"; - case GOING_AWAY -> "Going Away"; - case PROTOCOL_ERROR -> "Protocol Error"; - case UNSUPPORTED_DATA -> "Unsupported Data"; - case RESERVED -> "Reserved"; - case NO_STATUS_RECEIVED -> "No Status Received"; - case ABNORMAL_CLOSURE -> "Abnormal Closure"; - case INVALID_PAYLOAD_DATA -> "Invalid Payload Data"; - case POLICY_VIOLATION -> "Policy Violation"; - case MESSAGE_TOO_BIG -> "Message Too Big"; - case MANDATORY_EXTENSION -> "Mandatory Extension"; - case INTERNAL_ERROR -> "Internal Error"; - case TLS_HANDSHAKE -> "TLS Handshake"; - default -> { + switch (code) { + case NORMAL_CLOSURE: + return "Normal Closure"; + case GOING_AWAY: + return "Going Away"; + case PROTOCOL_ERROR: + return "Protocol Error"; + case UNSUPPORTED_DATA: + return "Unsupported Data"; + case RESERVED: + return "Reserved"; + case NO_STATUS_RECEIVED: + return "No Status Received"; + case ABNORMAL_CLOSURE: + return "Abnormal Closure"; + case INVALID_PAYLOAD_DATA: + return "Invalid Payload Data"; + case POLICY_VIOLATION: + return "Policy Violation"; + case MESSAGE_TOO_BIG: + return "Message Too Big"; + case MANDATORY_EXTENSION: + return "Mandatory Extension"; + case INTERNAL_ERROR: + return "Internal Error"; + case TLS_HANDSHAKE: + return "TLS Handshake"; + default: if (code >= 3000 && code < 4000) { - yield "Library/Framework Code (" + code + ")"; + return "Library/Framework Code (" + code + ")"; } else if (code >= 4000 && code < 5000) { - yield "Application Code (" + code + ")"; + return "Application Code (" + code + ")"; } - yield "Unknown (" + code + ")"; - } - }; + return "Unknown (" + code + ")"; + } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 54c5464..f5d3cf0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -245,15 +245,17 @@ private void handleRead() { } switch (opcode) { - case WebSocketOpcode.BINARY -> handler.onBinaryMessage(this, payload); - case WebSocketOpcode.PING -> { + case WebSocketOpcode.BINARY: + handler.onBinaryMessage(this, payload); + break; + case WebSocketOpcode.PING: try { writeFrame(WebSocketOpcode.PONG, payload, payload.length); } catch (IOException e) { LOG.error("Failed to send pong", e); } - } - case WebSocketOpcode.CLOSE -> { + break; + case WebSocketOpcode.CLOSE: { int code = WebSocketCloseCode.NORMAL_CLOSURE; if (payload.length >= 2) { code = ((payload[0] & 0xFF) << 8) | (payload[1] & 0xFF); @@ -265,6 +267,7 @@ private void handleRead() { } ClientHandler.this.running.set(false); isClosed = true; + break; } } } From 6452bd383d25850e92026891ec2f9c71b5bb690f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 13 Mar 2026 14:52:13 +0100 Subject: [PATCH 193/230] Fix awaitEmpty() race in InFlightWindow awaitEmpty() had a TOCTOU race between its error check and in-flight count check. When the I/O thread called fail() (setting lastError) and then acknowledgeUpTo() (draining the window to zero) before the flush thread was scheduled, the while loop exited on the count condition without re-entering the body to call checkError(). This caused flaky failures in testErrorPropagation_asyncMultipleBatchesInFlight. Add a final checkError() after the while loop. Correctness relies on the happens-before chain through the volatile highestAcked field: the I/O thread's lastError.set() precedes its highestAcked write, and the flush thread's highestAcked read precedes its lastError.get(), so the error is guaranteed to be visible. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/client/InFlightWindow.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java index 6447cb8..dae5daf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java @@ -287,6 +287,11 @@ public void awaitEmpty() { } } + // The I/O thread may have called fail() and then acknowledgeUpTo() + // before this thread was scheduled, draining the window while an + // error is pending. Check one final time after the window is empty. + checkError(); + LOG.debug("Window empty, all batches ACKed"); } finally { waitingForEmpty = null; From 8fb0553c37e9c23ab9f3f5b61b150e35f95131d2 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 13 Mar 2026 15:19:44 +0100 Subject: [PATCH 194/230] Refactor WebSocketSender API to unify connection methods Replaced `connectAsync` with an enhanced `connect` to simplify API usage and eliminate redundancy. Updated tests and benchmarks to align with the new unified connection method, ensuring compatibility and maintaining functionality. Adjusted default auto-flush and configuration parameters for better performance. --- .../main/java/io/questdb/client/Sender.java | 2 +- .../qwp/client/QwpWebSocketSender.java | 134 +++--------------- .../QwpWebSocketAckIntegrationTest.java | 12 +- 3 files changed, 30 insertions(+), 118 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index f63b92c..41abb75 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -885,7 +885,7 @@ public Sender build() { String wsAuthHeader = buildWebSocketAuthHeader(); - return QwpWebSocketSender.connectAsync( + return QwpWebSocketSender.connect( hosts.getQuick(0), ports.getQuick(0), tlsEnabled, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 0c19b45..5632e2c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -66,8 +66,8 @@ *

    * Configuration options: *

      - *
    • {@code autoFlushRows} - Maximum rows per batch (default: 500)
      • - *
    • {@code autoFlushBytes} - Maximum bytes per batch (default: 1MB)
      • + *
    • {@code autoFlushRows} - Maximum rows per batch (default: 1000)
      • + *
    • {@code autoFlushBytes} - Maximum bytes per batch (default: 128KB)
      • *
    • {@code autoFlushIntervalNanos} - Maximum age before auto-flush (default: 100ms)
      • *
    *

    @@ -189,7 +189,7 @@ private QwpWebSocketSender( /** * Creates a new sender and connects to the specified host and port. - * Uses synchronous mode for backward compatibility. + * Uses default auto-flush settings and in-flight window size. * * @param host server host * @param port server HTTP port (WebSocket upgrade happens on same port) @@ -200,8 +200,8 @@ public static QwpWebSocketSender connect(String host, int port) { } /** - * Creates a new sender with TLS and connects to the specified host and port. - * Uses synchronous mode with default auto-flush settings. + * Creates a new sender and connects to the specified host and port. + * Uses default auto-flush settings and in-flight window size. * * @param host server host * @param port server HTTP port @@ -210,13 +210,17 @@ public static QwpWebSocketSender connect(String host, int port) { */ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled) { return connect( - host, port, tlsEnabled, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS + host, port, tlsEnabled, + DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, + DEFAULT_IN_FLIGHT_WINDOW_SIZE, null ); } /** - * Creates a new sender with TLS and connects to the specified host and port. - * Uses synchronous mode with custom auto-flush settings. + * Creates a new sender with full configuration and connects. + *

    + * In-flight window size controls the flow behavior: 1 means synchronous (each batch + * waits for ACK), greater than 1 enables asynchronous pipelining with a background I/O thread. * * @param host server host * @param port server HTTP port @@ -224,93 +228,11 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl * @param autoFlushRows rows per batch (0 = no limit) * @param autoFlushBytes bytes per batch (0 = no limit) * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize max batches awaiting server ACK (1 = sync, default: 128) + * @param authorizationHeader HTTP Authorization header value, or null * @return connected sender */ public static QwpWebSocketSender connect( - String host, - int port, - boolean tlsEnabled, - int autoFlushRows, - int autoFlushBytes, - long autoFlushIntervalNanos - ) { - return connect(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, null); - } - - public static QwpWebSocketSender connect( - String host, - int port, - boolean tlsEnabled, - int autoFlushRows, - int autoFlushBytes, - long autoFlushIntervalNanos, - String authorizationHeader - ) { - QwpWebSocketSender sender = new QwpWebSocketSender( - host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - 1, // window=1 for sync behavior - authorizationHeader - ); - try { - sender.ensureConnected(); - } catch (Throwable t) { - sender.close(); - throw t; - } - return sender; - } - - /** - * Creates a new sender with async mode and custom configuration. - * - * @param host server host - * @param port server HTTP port - * @param tlsEnabled whether to use TLS - * @param autoFlushRows rows per batch (0 = no limit) - * @param autoFlushBytes bytes per batch (0 = no limit) - * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) - * @return connected sender - */ - public static QwpWebSocketSender connectAsync( - String host, - int port, - boolean tlsEnabled, - int autoFlushRows, - int autoFlushBytes, - long autoFlushIntervalNanos - ) { - return connectAsync( - host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, DEFAULT_IN_FLIGHT_WINDOW_SIZE - ); - } - - /** - * Creates a new sender with async mode and full configuration including flow control. - * - * @param host server host - * @param port server HTTP port - * @param tlsEnabled whether to use TLS - * @param autoFlushRows rows per batch (0 = no limit) - * @param autoFlushBytes bytes per batch (0 = no limit) - * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) - * @param inFlightWindowSize max batches awaiting server ACK (default: 8) - * @return connected sender - */ - public static QwpWebSocketSender connectAsync( - String host, - int port, - boolean tlsEnabled, - int autoFlushRows, - int autoFlushBytes, - long autoFlushIntervalNanos, - int inFlightWindowSize - ) { - return connectAsync( - host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, null - ); - } - - public static QwpWebSocketSender connectAsync( String host, int port, boolean tlsEnabled, @@ -321,7 +243,9 @@ public static QwpWebSocketSender connectAsync( String authorizationHeader ) { QwpWebSocketSender sender = new QwpWebSocketSender( - host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, authorizationHeader + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, authorizationHeader ); try { sender.ensureConnected(); @@ -332,20 +256,6 @@ public static QwpWebSocketSender connectAsync( return sender; } - /** - * Creates a new sender with async mode and default configuration. - * - * @param host server host - * @param port server HTTP port - * @param tlsEnabled whether to use TLS - * @return connected sender - */ - public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { - return connectAsync( - host, port, tlsEnabled, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS - ); - } - /** * Creates a sender without connecting. For testing only. *

    @@ -359,9 +269,10 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls */ public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { return new QwpWebSocketSender( - host, port, false, DEFAULT_BUFFER_SIZE, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, inFlightWindowSize, null + host, port, false, DEFAULT_BUFFER_SIZE, + DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, + inFlightWindowSize, null ); - // Note: does NOT call ensureConnected() } /** @@ -384,9 +295,10 @@ public static QwpWebSocketSender createForTesting( int inFlightWindowSize ) { return new QwpWebSocketSender( - host, port, false, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, null + host, port, false, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, null ); - // Note: does NOT call ensureConnected() } @Override diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java index d63ff1b..f8c0b25 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java @@ -55,8 +55,8 @@ public void testAsyncFlushFailsFastOnInvalidAckPayload() throws Exception { boolean errorCaught = false; long start = System.currentTimeMillis(); - try (QwpWebSocketSender sender = QwpWebSocketSender.connectAsync( - "localhost", port, false, 0, 0, 0)) { + try (QwpWebSocketSender sender = QwpWebSocketSender.connect( + "localhost", port, false, 0, 0, 0, QwpWebSocketSender.DEFAULT_IN_FLIGHT_WINDOW_SIZE, null)) { sender.table("test") .longColumn("value", 1) .atNow(); @@ -86,8 +86,8 @@ public void testAsyncFlushFailsFastOnServerClose() throws Exception { boolean errorCaught = false; long start = System.currentTimeMillis(); - try (QwpWebSocketSender sender = QwpWebSocketSender.connectAsync( - "localhost", port, false, 0, 0, 0)) { + try (QwpWebSocketSender sender = QwpWebSocketSender.connect( + "localhost", port, false, 0, 0, 0, QwpWebSocketSender.DEFAULT_IN_FLIGHT_WINDOW_SIZE, null)) { sender.table("test") .longColumn("value", 1) .atNow(); @@ -121,8 +121,8 @@ public void testFlushBlocksUntilAcked() throws Exception { server.start(); Assert.assertTrue("Server failed to start", server.awaitStart(5, TimeUnit.SECONDS)); - try (QwpWebSocketSender sender = QwpWebSocketSender.connectAsync( - "localhost", port, false, 0, 0, 0)) { + try (QwpWebSocketSender sender = QwpWebSocketSender.connect( + "localhost", port, false, 0, 0, 0, QwpWebSocketSender.DEFAULT_IN_FLIGHT_WINDOW_SIZE, null)) { sender.table("test") .longColumn("value", 42) From a38487c40ffc7cdb480567a842cb8aed8ed97f82 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 13 Mar 2026 16:55:44 +0100 Subject: [PATCH 195/230] remove blocking waits from websocket ACK drain --- .../qwp/client/QwpWebSocketSender.java | 10 +- .../qwp/client/WebSocketSendQueue.java | 69 +++++++---- .../qwp/client/WebSocketSendQueueTest.java | 107 ++++++++++++++++++ 3 files changed, 163 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index ff1c113..99a61a3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -400,8 +400,8 @@ public void close() { // Wait for all batches to be sent and acknowledged before closing if (sendQueue != null) { sendQueue.flush(); - } - if (inFlightWindow != null) { + sendQueue.awaitPendingAcks(); + } else if (inFlightWindow != null) { inFlightWindow.awaitEmpty(); } } else { @@ -581,7 +581,11 @@ public void flush() { sendQueue.flush(); // Wait for all in-flight batches to be acknowledged by the server - inFlightWindow.awaitEmpty(); + if (sendQueue != null) { + sendQueue.awaitPendingAcks(); + } else { + inFlightWindow.awaitEmpty(); + } LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); } else { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index d5c39b4..db01b07 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -64,6 +64,7 @@ */ public class WebSocketSendQueue implements QuietCloseable { + private static final int DRAIN_SPIN_TRIES = 16; public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); @@ -264,7 +265,6 @@ public boolean enqueue(MicrobatchBuffer buffer) { } } } - LOG.debug("Enqueued batch [id={}, bytes={}, rows={}]", buffer.getBatchId(), buffer.getBufferPos(), buffer.getRowCount()); return true; } @@ -282,7 +282,7 @@ public void flush() { long startTime = System.currentTimeMillis(); - // Wait under lock - I/O thread will notify when processingCount decrements + // Wait under lock until the queue becomes empty and no batch is being sent. synchronized (processingLock) { while (running) { // Atomically check: queue empty AND not processing @@ -290,19 +290,19 @@ public void flush() { break; // All done } + long remaining = enqueueTimeoutMs - (System.currentTimeMillis() - startTime); + if (remaining <= 0) { + throw new LineSenderException("Flush timeout after " + enqueueTimeoutMs + "ms, " + + "queue=" + getPendingSize() + ", processing=" + processingCount.get()); + } + try { - processingLock.wait(10); + processingLock.wait(remaining); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new LineSenderException("Interrupted while flushing", e); } - // Check timeout - if (System.currentTimeMillis() - startTime > enqueueTimeoutMs) { - throw new LineSenderException("Flush timeout after " + enqueueTimeoutMs + "ms, " + - "queue=" + getPendingSize() + ", processing=" + processingCount.get()); - } - // Check for errors checkError(); } @@ -314,6 +314,19 @@ public void flush() { LOG.debug("Flush complete"); } + /** + * Waits for all in-flight batches to be acknowledged. + */ + public void awaitPendingAcks() { + if (inFlightWindow == null) { + return; + } + + checkError(); + inFlightWindow.awaitEmpty(); + checkError(); + } + /** * Returns the last error that occurred in the I/O thread, or null if no error. */ @@ -387,6 +400,15 @@ private int getPendingSize() { return pendingBuffer == null ? 0 : 1; } + private int idleDuringDrain(int idleCycles) { + if (idleCycles < DRAIN_SPIN_TRIES) { + Thread.onSpinWait(); + return idleCycles + 1; + } + Thread.yield(); + return DRAIN_SPIN_TRIES; + } + /** * The main I/O loop that handles both sending batches and receiving ACKs. *

    @@ -394,20 +416,23 @@ private int getPendingSize() { *

      *
    • IDLE: block on processingLock.wait() until work arrives
      • *
    • ACTIVE: non-blocking poll queue, send batches, check for ACKs
      • - *
    • DRAINING: no batches but ACKs pending - poll for ACKs with short wait
      • + *
    • DRAINING: no batches but ACKs pending - poll for ACKs with non-blocking backoff
      • *
    */ private void ioLoop() { LOG.info("I/O loop started"); try { + int drainIdleCycles = 0; while (running || !isPendingEmpty()) { MicrobatchBuffer batch = null; boolean hasInFlight = (inFlightWindow != null && inFlightWindow.getInFlightCount() > 0); IoState state = computeState(hasInFlight); + boolean receivedAcks = false; switch (state) { case IDLE: + drainIdleCycles = 0; // Nothing to do - wait for work under lock synchronized (processingLock) { // Re-check under lock to avoid missed wakeup @@ -425,7 +450,7 @@ private void ioLoop() { case DRAINING: // Try to receive any pending ACKs (non-blocking) if (hasInFlight && client.isConnected()) { - tryReceiveAcks(); + receivedAcks = tryReceiveAcks(); } // Try to dequeue and send a batch @@ -452,15 +477,16 @@ private void ioLoop() { } } - // In DRAINING state with no work, short wait to avoid busy loop + // In DRAINING state with no work, stay non-blocking and use + // a simple spin/yield backoff. if (state == IoState.DRAINING && batch == null) { - synchronized (processingLock) { - try { - processingLock.wait(10); - } catch (InterruptedException e) { - if (!running) return; - } + if (receivedAcks) { + drainIdleCycles = 0; + } else { + drainIdleCycles = idleDuringDrain(drainIdleCycles); } + } else { + drainIdleCycles = 0; } break; } @@ -553,9 +579,11 @@ private void sendBatch(MicrobatchBuffer batch) { /** * Tries to receive ACKs from the server (non-blocking). */ - private void tryReceiveAcks() { + private boolean tryReceiveAcks() { + boolean received = false; try { while (client.tryReceiveFrame(responseHandler)) { + received = true; // Drain all buffered ACKs before returning to the I/O loop. } } catch (Exception e) { @@ -564,6 +592,7 @@ private void tryReceiveAcks() { failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); } } + return received; } /** @@ -571,7 +600,7 @@ private void tryReceiveAcks() { *
      *
    • IDLE: queue empty, no in-flight batches - can block waiting for work
      • *
    • ACTIVE: have batches to send - non-blocking loop
      • - *
    • DRAINING: queue empty but ACKs pending - poll for ACKs, short wait
      • + *
    • DRAINING: queue empty but ACKs pending - poll for ACKs with non-blocking backoff
      • *
    */ private enum IoState { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java index db97192..42bafb9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java @@ -30,6 +30,7 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.InFlightWindow; import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.MemoryTag; @@ -40,6 +41,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.*; @@ -266,6 +269,84 @@ public void testFlushFailsWhenServerClosesConnection() throws Exception { }); } + @Test + public void testAwaitPendingAcksKeepsDrainNonBlocking() throws Exception { + assertMemoryLeak(() -> { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + CountDownLatch secondBatchSent = new CountDownLatch(1); + AtomicBoolean deliverAcks = new AtomicBoolean(false); + AtomicInteger tryReceivePolls = new AtomicInteger(); + AtomicLong highestSent = new AtomicLong(-1); + AtomicReference errorRef = new AtomicReference<>(); + + try { + client.setSendBehavior((dataPtr, length) -> { + long sent = highestSent.incrementAndGet(); + if (sent == 1) { + secondBatchSent.countDown(); + } + }); + client.setReceiveBehavior((handler, timeout) -> { + throw new AssertionError("receiveFrame() must not be used while draining ACKs"); + }); + client.setTryReceiveBehavior(handler -> { + tryReceivePolls.incrementAndGet(); + if (deliverAcks.get()) { + long sent = highestSent.get(); + if (sent >= 0 && window.getInFlightCount() > 0) { + emitAck(handler, sent); + return true; + } + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + queue.enqueue(batch0); + queue.flush(); + + CountDownLatch finished = new CountDownLatch(1); + WebSocketSendQueue finalQueue = queue; + Thread waiter = new Thread(() -> { + try { + finalQueue.awaitPendingAcks(); + } catch (Throwable t) { + errorRef.set(t); + } finally { + finished.countDown(); + } + }); + waiter.start(); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (tryReceivePolls.get() == 0 && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertTrue("Expected non-blocking ACK polls while draining", tryReceivePolls.get() > 0); + + queue.enqueue(batch1); + assertTrue("I/O thread should still send new work while ACK drain is active", + secondBatchSent.await(1, TimeUnit.SECONDS)); + + deliverAcks.set(true); + + assertTrue("awaitPendingAcks should complete once ACK arrives", + finished.await(2, TimeUnit.SECONDS)); + assertNull(errorRef.get()); + assertEquals(0, window.getInFlightCount()); + } finally { + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + }); + } + private static void awaitThreadBlocked(Thread thread) throws InterruptedException { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (System.nanoTime() < deadline) { @@ -296,6 +377,18 @@ private static void emitBinary(WebSocketFrameHandler handler, byte[] payload) { } } + private static void emitAck(WebSocketFrameHandler handler, long sequence) { + WebSocketResponse response = WebSocketResponse.success(sequence); + int size = response.serializedSize(); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + response.writeTo(ptr); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + private static MicrobatchBuffer sealedBuffer(byte value) { MicrobatchBuffer buffer = new MicrobatchBuffer(64); buffer.writeByte(value); @@ -312,9 +405,14 @@ private interface TryReceiveBehavior { boolean tryReceive(WebSocketFrameHandler handler); } + private interface ReceiveBehavior { + boolean receive(WebSocketFrameHandler handler, int timeout); + } + private static class FakeWebSocketClient extends WebSocketClient { private volatile TryReceiveBehavior behavior = handler -> false; private volatile boolean connected = true; + private volatile ReceiveBehavior receiveBehavior = (handler, timeout) -> false; private volatile SendBehavior sendBehavior = (dataPtr, length) -> { }; @@ -346,6 +444,15 @@ public void setTryReceiveBehavior(TryReceiveBehavior behavior) { this.behavior = behavior; } + public void setReceiveBehavior(ReceiveBehavior receiveBehavior) { + this.receiveBehavior = receiveBehavior; + } + + @Override + public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) { + return receiveBehavior.receive(handler, timeout); + } + @Override public boolean tryReceiveFrame(WebSocketFrameHandler handler) { return behavior.tryReceive(handler); From 862798cf3fc99fd8f77354f4af5cf674d42f5a61 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Mon, 16 Mar 2026 16:26:50 +0100 Subject: [PATCH 196/230] Fix "an QWP" grammar in Javadoc comment Replace "an QWP" with "a QWP" in QwpColumnDef Javadoc. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index c7355ac..8405924 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -28,7 +28,7 @@ import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_CHAR; /** - * Represents a column definition in an QWP v1 schema. + * Represents a column definition in a QWP v1 schema. *

    * This class is immutable and safe for caching. */ From 893841b5e62df2f2361b04498833577ec3a8c53d Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 09:20:52 +0100 Subject: [PATCH 197/230] Replace /** with /*+ in license header opening lines Change the opening line of 60 license/copyright block comments from /*** to /*+*** so they are no longer parsed as Javadoc. The /** prefix triggers Javadoc formatting warnings on the copyright notice text, which are avoided by using /*+ instead. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/http/client/WebSocketClient.java | 2 +- .../client/cutlass/http/client/WebSocketClientFactory.java | 2 +- .../client/cutlass/http/client/WebSocketClientLinux.java | 2 +- .../questdb/client/cutlass/http/client/WebSocketClientOsx.java | 2 +- .../client/cutlass/http/client/WebSocketClientWindows.java | 2 +- .../client/cutlass/http/client/WebSocketFrameHandler.java | 2 +- .../questdb/client/cutlass/http/client/WebSocketSendBuffer.java | 2 +- .../client/cutlass/qwp/client/GlobalSymbolDictionary.java | 2 +- .../io/questdb/client/cutlass/qwp/client/InFlightWindow.java | 2 +- .../io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java | 2 +- .../questdb/client/cutlass/qwp/client/NativeBufferWriter.java | 2 +- .../io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java | 2 +- .../io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java | 2 +- .../java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java | 2 +- .../questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java | 2 +- .../questdb/client/cutlass/qwp/client/QwpWebSocketSender.java | 2 +- .../io/questdb/client/cutlass/qwp/client/WebSocketResponse.java | 2 +- .../questdb/client/cutlass/qwp/client/WebSocketSendQueue.java | 2 +- .../client/cutlass/qwp/protocol/OffHeapAppendMemory.java | 2 +- .../io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java | 2 +- .../io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java | 2 +- .../io/questdb/client/cutlass/qwp/protocol/QwpConstants.java | 2 +- .../questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java | 2 +- .../io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java | 2 +- .../io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java | 2 +- .../client/cutlass/qwp/websocket/WebSocketCloseCode.java | 2 +- .../client/cutlass/qwp/websocket/WebSocketFrameParser.java | 2 +- .../client/cutlass/qwp/websocket/WebSocketFrameWriter.java | 2 +- .../questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java | 2 +- .../main/java/io/questdb/client/std/CharSequenceIntHashMap.java | 2 +- core/src/main/java/io/questdb/client/std/SecureRnd.java | 2 +- .../client/test/cutlass/http/client/WebSocketClientTest.java | 2 +- .../questdb/client/test/cutlass/line/LineSenderBuilderTest.java | 2 +- .../test/cutlass/line/tcp/v4/QwpAllocationTestClient.java | 2 +- .../client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java | 2 +- .../test/cutlass/qwp/client/AsyncModeIntegrationTest.java | 2 +- .../test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java | 2 +- .../test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java | 2 +- .../client/test/cutlass/qwp/client/InFlightWindowTest.java | 2 +- .../test/cutlass/qwp/client/LineSenderBuilderUdpTest.java | 2 +- .../test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java | 2 +- .../client/test/cutlass/qwp/client/MicrobatchBufferTest.java | 2 +- .../client/test/cutlass/qwp/client/NativeBufferWriterTest.java | 2 +- .../test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java | 2 +- .../client/test/cutlass/qwp/client/QwpUdpSenderTest.java | 2 +- .../test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java | 2 +- .../client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java | 2 +- .../test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java | 2 +- .../client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java | 2 +- .../client/test/cutlass/qwp/client/WebSocketSendQueueTest.java | 2 +- .../test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java | 2 +- .../client/test/cutlass/qwp/protocol/QwpBitWriterTest.java | 2 +- .../client/test/cutlass/qwp/protocol/QwpColumnDefTest.java | 2 +- .../client/test/cutlass/qwp/protocol/QwpConstantsTest.java | 2 +- .../test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java | 2 +- .../client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java | 2 +- .../client/test/cutlass/qwp/protocol/QwpTableBufferTest.java | 2 +- .../client/test/cutlass/qwp/websocket/TestWebSocketServer.java | 2 +- .../test/cutlass/qwp/websocket/WebSocketFrameParserTest.java | 2 +- .../src/test/java/io/questdb/client/test/std/SecureRndTest.java | 2 +- 60 files changed, 60 insertions(+), 60 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 03b468b..6e38872 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java index 4972176..f48edd9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java index f4ac6ba..9ac4ef7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java index d34df7c..b0b257a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java index cdaec88..1e71699 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java index e3682f5..f0b8d8e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index d31044f..4b13481 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index b8c1dfe..7879845 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java index dae5daf..da56a1c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 38d29a3..dee78d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 9b979c9..90a2998 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java index 34347c6..0b8101c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index b3a5896..2a127fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index fcb2d78..e0d580c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index da62b16..56098cf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 99a61a3..6ae5105 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 99b7eba..b583b9c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index db01b07..e249351 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index 308a723..f546a30 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index c9ad763..675b3cc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index 8405924..88efb38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index df55c22..fef72eb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 3d8c104..994dfaa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 5105023..43d8675 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index af94111..f663134 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 0e70e0c..3a86bf6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index 6d4a7ac..ca6cab9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index 2f3c653..59e7598 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java index 8668644..74bb7bf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java index d56c181..29b4e39 100644 --- a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/std/SecureRnd.java b/core/src/main/java/io/questdb/client/std/SecureRnd.java index 2ceef88..91c2609 100644 --- a/core/src/main/java/io/questdb/client/std/SecureRnd.java +++ b/core/src/main/java/io/questdb/client/std/SecureRnd.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java index aa8d142..ba7c26d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java index 0935b64..2711592 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index bbf3c19..17a3d55 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index b7707f1..bcd4d15 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java index a2065a4..8d95212 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/AsyncModeIntegrationTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java index f24dc35..9079dbb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java index c5eb84e..bfb9f80 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java index 36d098d..5822d3e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java index 663d61e..5e8f191 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderUdpTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index c0c38c3..1505f1c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java index aebfc56..4d96e59 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 1f806a9..75ce11c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java index 1fa657e..90897c8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index b436ed0..68ba573 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java index f8c0b25..81524f1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketAckIntegrationTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index eab1c21..49adc6b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index 6e89f98..f96d2cf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 2fccdc5..f444918 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java index 42bafb9..91638d8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java index 8726bcf..8fae113 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java index 5ec8c60..2e539dc 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java index 2f5a3ac..b942dc3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java index f8654d5..c024457 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java index ab4d632..8fff334 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java index 129f56b..0a8326a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 02da14d..98e5ddb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index f5d3cf0..79ef4ce 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java index 4a5fed9..5f78906 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java index 6c04837..4dac697 100644 --- a/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java +++ b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ From 6fb4de07c9ee083b92f8b0e282e2ba10212e9ec6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 15:55:14 +0100 Subject: [PATCH 198/230] Replace CharSequence.isEmpty() with length check CharSequence.isEmpty() is a default method added in Java 15, but the client module targets Java 11. Replace all four call sites with the equivalent length() == 0 check to avoid NoSuchMethodError at runtime on Java 11. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/client/QwpUdpSender.java | 2 +- .../questdb/client/cutlass/qwp/client/QwpWebSocketSender.java | 4 ++-- .../client/cutlass/qwp/protocol/OffHeapAppendMemory.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index e0d580c..63fa612 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -778,7 +778,7 @@ private void beginColumnWrite(QwpTableBuffer.ColumnBuffer column, CharSequence c int columnIndex = column.getIndex(); ensureStagedColumnMarkCapacity(columnIndex + 1); if (stagedColumnMarks[columnIndex] == currentRowMark) { - if (columnName != null && columnName.isEmpty()) { + if (columnName != null && columnName.length() == 0) { throw new LineSenderException("designated timestamp already set for current row"); } throw new LineSenderException("column '" + columnName + "' already set for current row"); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 6ae5105..bfa7e0d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -486,7 +486,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { @Override public Sender decimalColumn(CharSequence name, CharSequence value) { - if (value == null || value.isEmpty()) return this; + if (value == null || value.length() == 0) return this; checkNotClosed(); checkTableSelected(); try { @@ -903,7 +903,7 @@ private void checkTableSelected() { private String checkedColumnName(CharSequence name) { if (name == null || !TableUtils.isValidColumnName(name, DEFAULT_MAX_NAME_LENGTH)) { - if (name == null || name.isEmpty()) { + if (name == null || name.length() == 0) { throw new LineSenderException("column name cannot be empty"); } if (name.length() > DEFAULT_MAX_NAME_LENGTH) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index f546a30..65a680b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -150,7 +150,7 @@ public void putShort(short value) { * Pre-ensures worst-case capacity to avoid per-byte checks. */ public void putUtf8(CharSequence value) { - if (value == null || value.isEmpty()) { + if (value == null || value.length() == 0) { return; } int len = value.length(); From e4d86dc1a584245584236a80665b60075b77d332 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 16:32:45 +0100 Subject: [PATCH 199/230] Fix resource leak in WebSocketClient subclass constructors If the platform I/O allocation (Kqueue, Epoll, or FDSet) throws after super() succeeds, the parent resources (socket, sendBuffer, controlFrameBuffer, recvBufPtr) were never freed. Wrap each subclass allocation in a try-catch that calls close() on failure, which frees both parent and subclass resources. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClientLinux.java | 13 +++++++++---- .../cutlass/http/client/WebSocketClientOsx.java | 13 +++++++++---- .../cutlass/http/client/WebSocketClientWindows.java | 9 +++++++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java index 9ac4ef7..0533a10 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java @@ -39,10 +39,15 @@ public class WebSocketClientLinux extends WebSocketClient { public WebSocketClientLinux(HttpClientConfiguration configuration, SocketFactory socketFactory) { super(configuration, socketFactory); - epoll = new Epoll( - configuration.getEpollFacade(), - configuration.getWaitQueueCapacity() - ); + try { + epoll = new Epoll( + configuration.getEpollFacade(), + configuration.getWaitQueueCapacity() + ); + } catch (Throwable t) { + close(); + throw t; + } } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java index b0b257a..b26b116 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java @@ -38,10 +38,15 @@ public class WebSocketClientOsx extends WebSocketClient { public WebSocketClientOsx(HttpClientConfiguration configuration, SocketFactory socketFactory) { super(configuration, socketFactory); - this.kqueue = new Kqueue( - configuration.getKQueueFacade(), - configuration.getWaitQueueCapacity() - ); + try { + this.kqueue = new Kqueue( + configuration.getKQueueFacade(), + configuration.getWaitQueueCapacity() + ); + } catch (Throwable t) { + close(); + throw t; + } } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java index 1e71699..3d00f82 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java @@ -40,8 +40,13 @@ public class WebSocketClientWindows extends WebSocketClient { public WebSocketClientWindows(HttpClientConfiguration configuration, SocketFactory socketFactory) { super(configuration, socketFactory); - this.fdSet = new FDSet(configuration.getWaitQueueCapacity()); - this.sf = configuration.getSelectFacade(); + try { + this.fdSet = new FDSet(configuration.getWaitQueueCapacity()); + this.sf = configuration.getSelectFacade(); + } catch (Throwable t) { + close(); + throw t; + } } @Override From 8d8713ed235f86ad542653f8f4ac08b2707bb4f7 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 16:44:11 +0100 Subject: [PATCH 200/230] Track pending bytes incrementally in sendRow() Replace the O(N*M) getPendingBytes() scan that iterated all table buffers and their columns on every row commit. Instead, maintain a running pendingBytes total by measuring the delta from the current table buffer before/after nextRow(). Reset the counter at the same three points where pendingRowCount resets. This reduces the per-row cost of shouldAutoFlush() from O(N*M) to O(M) for the single active table. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bfa7e0d..8dd2b41 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -145,6 +145,7 @@ public class QwpWebSocketSender implements Sender { // Batch sequence counter (must match server's messageSequence) private long nextBatchSequence = 0; // Async mode: pending row tracking + private long pendingBytes; private int pendingRowCount; private boolean sawBinaryAck; private WebSocketSendQueue sendQueue; @@ -758,6 +759,7 @@ public void reset() { buf.reset(); } } + pendingBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; currentTableBuffer = null; @@ -1071,6 +1073,7 @@ private void flushPendingRows() { } // Reset pending count + pendingBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; } @@ -1150,6 +1153,7 @@ private void flushSync() { } // Reset pending row tracking + pendingBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; @@ -1157,18 +1161,7 @@ private void flushSync() { } private long getPendingBytes() { - long bytes = 0; - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence key = keys.getQuick(i); - if (key != null) { - QwpTableBuffer tb = tableBuffers.get(key); - if (tb != null) { - bytes += tb.getBufferedBytes(); - } - } - } - return bytes; + return pendingBytes; } /** @@ -1226,7 +1219,9 @@ private void sealAndSwapBuffer() { */ private void sendRow() { ensureConnected(); + long bytesBefore = currentTableBuffer.getBufferedBytes(); currentTableBuffer.nextRow(); + pendingBytes += currentTableBuffer.getBufferedBytes() - bytesBefore; // Both modes: accumulate rows, don't encode yet if (pendingRowCount == 0) { From 65270d693e59a4f5c4b6e59f02de38dabb97d2a4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 16:46:21 +0100 Subject: [PATCH 201/230] Fix integer overflow in ensureCapacity() doubling NativeSegmentList.ensureCapacity() doubles newCapacity in a loop until it reaches the required size. When newCapacity exceeds Integer.MAX_VALUE / 2, the multiplication overflows to negative, causing the while loop to run infinitely. Guard against this by checking before each doubling and falling back to the exact required capacity when doubling would overflow. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/cutlass/qwp/client/NativeSegmentList.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java index f30bc4c..9d1f878 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java @@ -105,6 +105,10 @@ private void ensureCapacity(int required) { int newCapacity = capacity; while (newCapacity < required) { + if (newCapacity > Integer.MAX_VALUE / 2) { + newCapacity = required; + break; + } newCapacity *= 2; } From decf7973723d48c30210c4bebc4d21c57e487688 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 16:49:55 +0100 Subject: [PATCH 202/230] Fix buffer0 leak in QwpWebSocketSender constructor If the second MicrobatchBuffer allocation (buffer1) throws, the already-allocated buffer0 leaks its native memory. Neither connect() nor createForTesting() catch constructor failures, so the cleanup must happen in the constructor itself. Wrap the buffer1 allocation in a try-catch that closes buffer0 before re-throwing. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8dd2b41..16b0beb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -183,7 +183,12 @@ private QwpWebSocketSender( if (inFlightWindowSize > 1) { int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); this.buffer0 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); - this.buffer1 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + try { + this.buffer1 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + } catch (Throwable t) { + buffer0.close(); + throw t; + } this.activeBuffer = buffer0; } } From 50d3d0a36f9baf063db0f2543e258aade526510b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 16:26:21 +0100 Subject: [PATCH 203/230] Fix native memory leak in QwpUdpSender constructor Move NativeSegmentList, NativeBufferWriter, and SegmentedNativeBufferWriter allocations from field initializers into the constructor body, wrapped in a try-catch. If any allocation or the UdpLineChannel/HashMap construction throws, the catch block frees all previously allocated native resources via Misc.free() before re-throwing. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/QwpUdpSender.java | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 63fa612..aa3e8d1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -37,6 +37,7 @@ import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Misc; import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.std.bytes.DirectByteSlice; @@ -69,10 +70,10 @@ public class QwpUdpSender implements Sender { private static final int VARINT_INT_UPPER_BOUND = 5; private final UdpLineChannel channel; private final QwpColumnWriter columnWriter = new QwpColumnWriter(); - private final NativeSegmentList datagramSegments = new NativeSegmentList(); - private final NativeBufferWriter headerBuffer = new NativeBufferWriter(); + private final NativeSegmentList datagramSegments; + private final NativeBufferWriter headerBuffer; private final int maxDatagramSize; - private final SegmentedNativeBufferWriter payloadWriter = new SegmentedNativeBufferWriter(); + private final SegmentedNativeBufferWriter payloadWriter; private final CharSequenceObjHashMap tableBuffers; private final CharSequenceObjHashMap tableHeadroomStates; private final boolean trackDatagramEstimate; @@ -105,9 +106,28 @@ public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int } public QwpUdpSender(NetworkFacade nf, int interfaceIPv4, int sendToAddress, int port, int ttl, int maxDatagramSize) { - this.channel = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); - this.tableHeadroomStates = new CharSequenceObjHashMap<>(); - this.tableBuffers = new CharSequenceObjHashMap<>(); + NativeSegmentList segments = null; + NativeBufferWriter header = null; + SegmentedNativeBufferWriter payload = null; + UdpLineChannel ch = null; + try { + segments = new NativeSegmentList(); + header = new NativeBufferWriter(); + payload = new SegmentedNativeBufferWriter(); + ch = new UdpLineChannel(nf, interfaceIPv4, sendToAddress, port, ttl); + this.tableHeadroomStates = new CharSequenceObjHashMap<>(); + this.tableBuffers = new CharSequenceObjHashMap<>(); + } catch (Throwable t) { + Misc.free(ch); + Misc.free(payload); + Misc.free(header); + Misc.free(segments); + throw t; + } + this.channel = ch; + this.datagramSegments = segments; + this.headerBuffer = header; + this.payloadWriter = payload; this.maxDatagramSize = maxDatagramSize; this.trackDatagramEstimate = maxDatagramSize > 0; } From d76aa02f6a4483745b5a269c5f72ae4ac2badb9d Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 16:44:14 +0100 Subject: [PATCH 204/230] Remove redundant toString() in string/symbol columns Pass CharSequence directly through the symbol and string column paths instead of eagerly converting to String. In stringColumn(), addString() already copies character data immediately via putUtf8(), so the toString() was pure waste. In the symbol path, propagate CharSequence through getOrAddGlobalSymbol() and addSymbolWithGlobalId() down to the dictionary lookup. GlobalSymbolDictionary.getOrAddSymbol() now accepts CharSequence and only calls toString() when a symbol is new and must be stored. CharSequenceIntHashMap.get() already accepts CharSequence for lookups, so repeated symbols incur zero allocation. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/GlobalSymbolDictionary.java | 9 +++++---- .../cutlass/qwp/client/QwpWebSocketSender.java | 4 ++-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 14 ++++++-------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 7879845..3b4c9a9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -94,7 +94,7 @@ public int getId(String symbol) { * @return the global ID for this symbol (>= 0) * @throws IllegalArgumentException if symbol is null */ - public int getOrAddSymbol(String symbol) { + public int getOrAddSymbol(CharSequence symbol) { if (symbol == null) { throw new IllegalArgumentException("symbol cannot be null"); } @@ -104,10 +104,11 @@ public int getOrAddSymbol(String symbol) { return existingId; } - // Assign new ID + // Assign new ID — toString() only for new symbols that must be stored + String symbolStr = symbol.toString(); int newId = idToSymbol.size(); - symbolToId.put(symbol, newId); - idToSymbol.add(symbol); + symbolToId.put(symbolStr, newId); + idToSymbol.add(symbolStr); return newId; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 16b0beb..17acfb3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -636,7 +636,7 @@ public int getMaxSentSymbolId() { * @param symbol the symbol value to register * @return the global symbol ID */ - public int getOrAddGlobalSymbol(String symbol) { + public int getOrAddGlobalSymbol(CharSequence symbol) { int globalId = globalSymbolDictionary.getOrAddSymbol(symbol); if (globalId > currentBatchMaxSymbolId) { currentBatchMaxSymbolId = globalId; @@ -801,7 +801,7 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_STRING, true); - col.addString(value != null ? value.toString() : null); + col.addString(value); return this; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index f663134..71010e3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -78,7 +78,7 @@ public QwpTableBuffer(String tableName) { /** * Use this constructor overload to allow writing to a symbol column. * {@link ColumnBuffer#addSymbol(CharSequence)} needs the sender to - * call {@link QwpWebSocketSender#getOrAddGlobalSymbol(String)}, registering + * call {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}, registering * the symbol in the global dictionary shared with the server. */ public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { @@ -1063,9 +1063,8 @@ public void addSymbol(CharSequence value) { return; } if (sender != null) { - String symbolValue = value.toString(); - int globalId = sender.getOrAddGlobalSymbol(symbolValue); - addSymbolWithGlobalId(symbolValue, globalId); + int globalId = sender.getOrAddGlobalSymbol(value); + addSymbolWithGlobalId(value, globalId); return; } ensureNullBitmapCapacity(); @@ -1092,9 +1091,8 @@ public void addSymbolUtf8(long ptr, int len) { throw new AssertionError("unreachable"); } if (sender != null) { - String symbolValue = lookupSink.toString(); - int globalId = sender.getOrAddGlobalSymbol(symbolValue); - addSymbolWithGlobalId(symbolValue, globalId); + int globalId = sender.getOrAddGlobalSymbol(lookupSink); + addSymbolWithGlobalId(lookupSink, globalId); return; } ensureNullBitmapCapacity(); @@ -1104,7 +1102,7 @@ public void addSymbolUtf8(long ptr, int len) { size++; } - public void addSymbolWithGlobalId(String value, int globalId) { + public void addSymbolWithGlobalId(CharSequence value, int globalId) { if (value == null) { addNull(); return; From 61980ee727bf8d249176d3761dbeba971b77331d Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Tue, 17 Mar 2026 16:57:56 +0100 Subject: [PATCH 205/230] Fix WebSocket mask key byte order WebSocketFrameWriter.writeHeader() wrote the mask key via Unsafe.putInt() in native byte order. maskPayload() also extracted mask bytes in native order. RFC 6455 specifies network (big-endian) byte order for the mask key on the wire. writeHeader() now writes the 4 mask key bytes individually in big-endian order. maskPayload() converts to native order for bulk XOR and extracts per-byte mask in big-endian order. Co-Authored-By: Claude Opus 4.6 --- .../qwp/websocket/WebSocketFrameWriter.java | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index 59e7598..07ba22b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -26,6 +26,8 @@ import io.questdb.client.std.Unsafe; +import java.nio.ByteOrder; + /** * Zero-allocation WebSocket frame writer. * Writes WebSocket frames according to RFC 6455. @@ -37,6 +39,7 @@ public final class WebSocketFrameWriter { // Frame header bits private static final int FIN_BIT = 0x80; + private static final boolean IS_BIG_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN; private static final int MASK_BIT = 0x80; private WebSocketFrameWriter() { @@ -70,9 +73,13 @@ public static int headerSize(long payloadLength, boolean masked) { * @param maskKey the 4-byte mask key */ public static void maskPayload(long buf, long len, int maskKey) { - // Process 8 bytes at a time when possible + // maskKey is in big-endian convention: MSB = wire byte 0 = mask byte for position 0. + // For bulk XOR via getInt/getLong (native byte order), convert to native order + // so that memory position 0 XORs with mask byte 0, position 1 with mask byte 1, etc. + int nativeMask = IS_BIG_ENDIAN ? maskKey : Integer.reverseBytes(maskKey); + long longMask = ((long) nativeMask << 32) | (nativeMask & 0xFFFFFFFFL); + long i = 0; - long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); // Process 8-byte chunks while (i + 8 <= len) { @@ -84,14 +91,14 @@ public static void maskPayload(long buf, long len, int maskKey) { // Process 4-byte chunk if remaining if (i + 4 <= len) { int value = Unsafe.getUnsafe().getInt(buf + i); - Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); + Unsafe.getUnsafe().putInt(buf + i, value ^ nativeMask); i += 4; } - // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation + // Process remaining bytes - extract mask byte in big-endian order while (i < len) { byte b = Unsafe.getUnsafe().getByte(buf + i); - int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF; + int maskByte = (maskKey >>> ((3 - ((int) i & 3)) << 3)) & 0xFF; Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); i++; } @@ -144,7 +151,11 @@ public static int writeHeader(long buf, boolean fin, int opcode, long payloadLen */ public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { int offset = writeHeader(buf, fin, opcode, payloadLength, true); - Unsafe.getUnsafe().putInt(buf + offset, maskKey); + // Write mask key in network byte order (big-endian) per RFC 6455 + Unsafe.getUnsafe().putByte(buf + offset, (byte) (maskKey >>> 24)); + Unsafe.getUnsafe().putByte(buf + offset + 1, (byte) (maskKey >>> 16)); + Unsafe.getUnsafe().putByte(buf + offset + 2, (byte) (maskKey >>> 8)); + Unsafe.getUnsafe().putByte(buf + offset + 3, (byte) maskKey); return offset + 4; } } From 34b24390ae8c404d97331aff9b11fb14bdecda1f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 18 Mar 2026 09:51:31 +0100 Subject: [PATCH 206/230] Fix resource leaks in QwpWebSocketSender Close the encoder's native memory when MicrobatchBuffer allocation fails in the constructor. Previously only buffer0 was cleaned up, leaving the NativeBufferWriter leaked. In ensureConnected(), close the WebSocket client if WebSocketSendQueue construction fails (e.g. Thread.start() throws OOM), preventing a dangling socket and I/O thread. In flushSync(), fail the in-flight window entry when sendBinary() throws, so close() does not hang on awaitEmpty() waiting for an ACK that will never arrive. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 17acfb3..19cfe1d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -182,11 +182,14 @@ private QwpWebSocketSender( // Initialize double-buffering if async mode (window > 1) if (inFlightWindowSize > 1) { int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); - this.buffer0 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); try { + this.buffer0 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); this.buffer1 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); } catch (Throwable t) { - buffer0.close(); + if (buffer0 != null) { + buffer0.close(); + } + encoder.close(); throw t; } this.activeBuffer = buffer0; @@ -980,9 +983,16 @@ private void ensureConnected() { // Initialize send queue for async mode (window > 1) // The send queue handles both sending AND receiving (single I/O thread) if (inFlightWindowSize > 1) { - sendQueue = new WebSocketSendQueue(client, inFlightWindow, - WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, - WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); + try { + sendQueue = new WebSocketSendQueue(client, inFlightWindow, + WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, + WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); + } catch (Throwable t) { + inFlightWindow = null; + client.close(); + client = null; + throw new LineSenderException("Failed to start I/O thread for " + host + ":" + port, t); + } } // Sync mode (window=1): no send queue - we send and read ACKs synchronously @@ -1134,8 +1144,18 @@ private void flushSync() { LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), currentBatchMaxSymbolId, useSchemaRef); - // Send over WebSocket - client.sendBinary(buffer.getBufferPtr(), messageSize); + // Send over WebSocket and fail the in-flight entry if send throws, + // so close() does not hang waiting for an ACK that will never arrive. + try { + client.sendBinary(buffer.getBufferPtr(), messageSize); + } catch (LineSenderException e) { + failExpectedIfNeeded(batchSequence, e); + throw e; + } catch (Throwable t) { + LineSenderException error = new LineSenderException("Failed to send batch " + batchSequence, t); + failExpectedIfNeeded(batchSequence, error); + throw error; + } // Wait for ACK synchronously waitForAck(batchSequence); From 60d578d56b4dbcb58781594ac32c95fc56f67509 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 18 Mar 2026 09:44:35 +0100 Subject: [PATCH 207/230] Rename SEGMENT_SIZE and reorder methods Rename SEGMENT_SIZE constant to ENTRY_SIZE for clarity in NativeSegmentList. Reorder methods alphabetically: move close() and ensureCapacity() before add() to match the project's member-ordering convention. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/NativeSegmentList.java | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java index 9d1f878..91d4eab 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeSegmentList.java @@ -28,7 +28,7 @@ import io.questdb.client.std.Unsafe; final class NativeSegmentList implements QuietCloseable { - static final int SEGMENT_SIZE = 16; + static final int ENTRY_SIZE = 16; private int capacity; private long ptr; @@ -41,7 +41,41 @@ final class NativeSegmentList implements QuietCloseable { NativeSegmentList(int initialCapacity) { this.capacity = Math.max(initialCapacity, 4); - this.ptr = Unsafe.malloc((long) capacity * SEGMENT_SIZE, MemoryTag.NATIVE_DEFAULT); + this.ptr = Unsafe.malloc((long) capacity * ENTRY_SIZE, MemoryTag.NATIVE_DEFAULT); + } + + @Override + public void close() { + if (ptr != 0) { + Unsafe.free(ptr, (long) capacity * ENTRY_SIZE, MemoryTag.NATIVE_DEFAULT); + ptr = 0; + capacity = 0; + size = 0; + totalLength = 0; + } + } + + private void ensureCapacity(int required) { + if (required <= capacity) { + return; + } + + int newCapacity = capacity; + while (newCapacity < required) { + if (newCapacity > Integer.MAX_VALUE / 2) { + newCapacity = required; + break; + } + newCapacity *= 2; + } + + ptr = Unsafe.realloc( + ptr, + (long) capacity * ENTRY_SIZE, + (long) newCapacity * ENTRY_SIZE, + MemoryTag.NATIVE_DEFAULT + ); + capacity = newCapacity; } void add(long address, long length) { @@ -49,7 +83,7 @@ void add(long address, long length) { return; } ensureCapacity(size + 1); - long segmentPtr = ptr + (long) size * SEGMENT_SIZE; + long segmentPtr = ptr + (long) size * ENTRY_SIZE; Unsafe.getUnsafe().putLong(segmentPtr, address); Unsafe.getUnsafe().putLong(segmentPtr + 8, length); size++; @@ -63,8 +97,8 @@ void appendFrom(NativeSegmentList other) { ensureCapacity(size + other.size); Unsafe.getUnsafe().copyMemory( other.ptr, - ptr + (long) size * SEGMENT_SIZE, - (long) other.size * SEGMENT_SIZE + ptr + (long) size * ENTRY_SIZE, + (long) other.size * ENTRY_SIZE ); size += other.size; totalLength += other.totalLength; @@ -82,42 +116,8 @@ long getTotalLength() { return totalLength; } - @Override - public void close() { - if (ptr != 0) { - Unsafe.free(ptr, (long) capacity * SEGMENT_SIZE, MemoryTag.NATIVE_DEFAULT); - ptr = 0; - capacity = 0; - size = 0; - totalLength = 0; - } - } - void reset() { size = 0; totalLength = 0; } - - private void ensureCapacity(int required) { - if (required <= capacity) { - return; - } - - int newCapacity = capacity; - while (newCapacity < required) { - if (newCapacity > Integer.MAX_VALUE / 2) { - newCapacity = required; - break; - } - newCapacity *= 2; - } - - ptr = Unsafe.realloc( - ptr, - (long) capacity * SEGMENT_SIZE, - (long) newCapacity * SEGMENT_SIZE, - MemoryTag.NATIVE_DEFAULT - ); - capacity = newCapacity; - } } From 5d041c53727ff99979b22a5c33fbd8879ded587f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 18 Mar 2026 10:41:24 +0100 Subject: [PATCH 208/230] Avoid per-byte ensureCapacity in UTF-8 encoding Extract encodeUtf8() that writes directly to native memory via Unsafe.putByte on a running address pointer, with a single upfront ensureCapacity call for the full UTF-8 length. Both putUtf8() and putString() now compute utf8Length once, reserve capacity once, then delegate to encodeUtf8(). This also eliminates the double string scan that putString() previously performed (utf8Length + putUtf8's per-char loop with per-byte ensureCapacity). Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/NativeBufferWriter.java | 65 +++++++++++-------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 90a2998..18cafdf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -243,7 +243,8 @@ public void putString(String value) { int utf8Len = utf8Length(value); putVarint(utf8Len); - putUtf8(value); + ensureCapacity(utf8Len); + encodeUtf8(value); } /** @@ -254,33 +255,9 @@ public void putUtf8(String value) { if (value == null || value.isEmpty()) { return; } - for (int i = 0, n = value.length(); i < n; i++) { - char c = value.charAt(i); - if (c < 0x80) { - putByte((byte) c); - } else if (c < 0x800) { - putByte((byte) (0xC0 | (c >> 6))); - putByte((byte) (0x80 | (c & 0x3F))); - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { - char c2 = value.charAt(++i); - if (Character.isLowSurrogate(c2)) { - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - putByte((byte) (0xF0 | (codePoint >> 18))); - putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - putByte((byte) (0x80 | (codePoint & 0x3F))); - } else { - putByte((byte) '?'); - i--; - } - } else if (Character.isSurrogate(c)) { - putByte((byte) '?'); - } else { - putByte((byte) (0xE0 | (c >> 12))); - putByte((byte) (0x80 | ((c >> 6) & 0x3F))); - putByte((byte) (0x80 | (c & 0x3F))); - } - } + int utf8Len = utf8Length(value); + ensureCapacity(utf8Len); + encodeUtf8(value); } /** @@ -314,4 +291,36 @@ public void skip(int bytes) { ensureCapacity(bytes); position += bytes; } + + private void encodeUtf8(String value) { + long addr = bufferPtr + position; + for (int i = 0, n = value.length(); i < n; i++) { + char c = value.charAt(i); + if (c < 0x80) { + Unsafe.getUnsafe().putByte(addr++, (byte) c); + } else if (c < 0x800) { + Unsafe.getUnsafe().putByte(addr++, (byte) (0xC0 | (c >> 6))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + char c2 = value.charAt(++i); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + Unsafe.getUnsafe().putByte(addr++, (byte) (0xF0 | (codePoint >> 18))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | ((codePoint >> 12) & 0x3F))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | ((codePoint >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | (codePoint & 0x3F))); + } else { + Unsafe.getUnsafe().putByte(addr++, (byte) '?'); + i--; + } + } else if (Character.isSurrogate(c)) { + Unsafe.getUnsafe().putByte(addr++, (byte) '?'); + } else { + Unsafe.getUnsafe().putByte(addr++, (byte) (0xE0 | (c >> 12))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | ((c >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(addr++, (byte) (0x80 | (c & 0x3F))); + } + } + position = (int) (addr - bufferPtr); + } } From 1d3300609cf92968abda69a5ab936ad0ac366910 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 19 Mar 2026 08:20:57 +0100 Subject: [PATCH 209/230] Case-insensitive column names With case-sensitive names, client would add the same column twice if the user used the same name in different case. --- .../client/cutlass/qwp/protocol/QwpTableBuffer.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 71010e3..03ae2de 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -32,6 +32,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.Chars; +import io.questdb.client.std.LowerCaseAsciiCharSequenceIntHashMap; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -58,7 +59,7 @@ */ public class QwpTableBuffer implements QuietCloseable { - private final CharSequenceIntHashMap columnNameToIndex; + private final LowerCaseAsciiCharSequenceIntHashMap columnNameToIndex; private final ObjList columns; private final QwpWebSocketSender sender; private final String tableName; @@ -85,7 +86,7 @@ public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { this.tableName = tableName; this.sender = sender; this.columns = new ObjList<>(); - this.columnNameToIndex = new CharSequenceIntHashMap(); + this.columnNameToIndex = new LowerCaseAsciiCharSequenceIntHashMap(); this.rowCount = 0; this.schemaHash = 0; this.schemaHashComputed = false; @@ -360,7 +361,7 @@ private ColumnBuffer lookupColumn(CharSequence name, byte type) { int n = columns.size(); if (columnAccessCursor < n) { ColumnBuffer candidate = fastColumns[columnAccessCursor]; - if (Chars.equals(candidate.name, name)) { + if (Chars.equalsIgnoreCase(candidate.name, name)) { columnAccessCursor++; assertColumnType(name, type, candidate); return candidate; @@ -369,7 +370,7 @@ private ColumnBuffer lookupColumn(CharSequence name, byte type) { // Slow path: hash map lookup int idx = columnNameToIndex.get(name); - if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + if (idx >= 0) { ColumnBuffer existing = columns.get(idx); assertColumnType(name, type, existing); return existing; From 7fbfff581b65f1e1b9cbf110e0499f60e602b091 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 19 Mar 2026 08:21:42 +0100 Subject: [PATCH 210/230] Pre-pad new columns for existing rows QwpTableBuffer.createColumn() created new column buffers with size = 0 regardless of how many rows were already committed. When the caller added a value for the current row, it landed at position 0 (the first row's slot). The subsequent nextRow() call padded nulls after the value, misaligning the column data with all other columns. The fix pre-pads the new column with nulls for all committed rows so the next value lands at the correct row position. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 03ae2de..5a4d8fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -351,6 +351,11 @@ private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable fastColumns = newArr; } fastColumns[index] = col; + // Pre-pad with nulls for already-committed rows so the next + // value the caller adds lands at the correct row position. + for (int r = 0; r < rowCount; r++) { + col.addNull(); + } schemaHashComputed = false; columnDefsCacheValid = false; return col; From a55d3b9dcc981a0e9dbc79e10f626c29f178c95b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 19 Mar 2026 09:34:41 +0100 Subject: [PATCH 211/230] Fix duplicate column behavior When user would add a value to the same column column twice for the current row, client would write that as the value for the next row, breaking the invariant that all cols have the same length. Fix is to align behavior with ILP: first write wins, others silently ignored. --- .../qwp/client/QwpWebSocketSender.java | 108 +++++++++++++----- .../cutlass/qwp/protocol/QwpTableBuffer.java | 14 ++- 2 files changed, 92 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 19cfe1d..5df7b45 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -343,7 +343,9 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_BOOLEAN, false); - col.addBoolean(value); + if (col != null) { + col.addBoolean(value); + } return this; } @@ -363,7 +365,9 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_BYTE, false); - col.addByte(value); + if (col != null) { + col.addByte(value); + } return this; } @@ -388,7 +392,9 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_CHAR, false); - col.addShort((short) value); + if (col != null) { + col.addShort((short) value); + } return this; } @@ -469,7 +475,9 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL64, true); - col.addDecimal64(value); + if (col != null) { + col.addDecimal64(value); + } return this; } @@ -479,7 +487,9 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL128, true); - col.addDecimal128(value); + if (col != null) { + col.addDecimal128(value); + } return this; } @@ -489,7 +499,9 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL256, true); - col.addDecimal256(value); + if (col != null) { + col.addDecimal256(value); + } return this; } @@ -501,7 +513,9 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { try { currentDecimal256.ofString(value); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL256, true); - col.addDecimal256(currentDecimal256); + if (col != null) { + col.addDecimal256(currentDecimal256); + } } catch (Exception e) { throw new LineSenderException("Failed to parse decimal value: " + value, e); } @@ -514,7 +528,9 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); + if (col != null) { + col.addDoubleArray(values); + } return this; } @@ -524,7 +540,9 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); + if (col != null) { + col.addDoubleArray(values); + } return this; } @@ -534,7 +552,9 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); + if (col != null) { + col.addDoubleArray(values); + } return this; } @@ -544,7 +564,9 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(array); + if (col != null) { + col.addDoubleArray(array); + } return this; } @@ -553,7 +575,9 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_DOUBLE, true); - col.addDouble(value); + if (col != null) { + col.addDouble(value); + } return this; } @@ -568,7 +592,9 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_FLOAT, true); - col.addFloat(value); + if (col != null) { + col.addFloat(value); + } return this; } @@ -678,7 +704,9 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_INT, true); - col.addInt(value); + if (col != null) { + col.addInt(value); + } return this; } @@ -703,7 +731,9 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG256, true); - col.addLong256(l0, l1, l2, l3); + if (col != null) { + col.addLong256(l0, l1, l2, l3); + } return this; } @@ -713,7 +743,9 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); - col.addLongArray(values); + if (col != null) { + col.addLongArray(values); + } return this; } @@ -723,7 +755,9 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); - col.addLongArray(values); + if (col != null) { + col.addLongArray(values); + } return this; } @@ -733,7 +767,9 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); - col.addLongArray(values); + if (col != null) { + col.addLongArray(values); + } return this; } @@ -743,7 +779,9 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); - col.addLongArray(array); + if (col != null) { + col.addLongArray(array); + } return this; } @@ -752,7 +790,9 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG, true); - col.addLong(value); + if (col != null) { + col.addLong(value); + } return this; } @@ -795,7 +835,9 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_SHORT, false); - col.addShort(value); + if (col != null) { + col.addShort(value); + } return this; } @@ -804,7 +846,9 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_STRING, true); - col.addString(value); + if (col != null) { + col.addString(value); + } return this; } @@ -813,7 +857,9 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_SYMBOL, true); - col.addSymbol(value); + if (col != null) { + col.addSymbol(value); + } return this; } @@ -844,11 +890,15 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkTableSelected(); if (unit == ChronoUnit.NANOS) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP_NANOS, true); - col.addLong(value); + if (col != null) { + col.addLong(value); + } } else { long micros = toMicros(value, unit); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP, true); - col.addLong(micros); + if (col != null) { + col.addLong(micros); + } } return this; } @@ -859,7 +909,9 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP, true); - col.addLong(micros); + if (col != null) { + col.addLong(micros); + } return this; } @@ -875,7 +927,9 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) checkNotClosed(); checkTableSelected(); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_UUID, true); - col.addUuid(hi, lo); + if (col != null) { + col.addUuid(hi, lo); + } return this; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 5a4d8fa..d18ec82 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -32,11 +32,11 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.Chars; -import io.questdb.client.std.LowerCaseAsciiCharSequenceIntHashMap; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Decimals; +import io.questdb.client.std.LowerCaseAsciiCharSequenceIntHashMap; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.NumericException; import io.questdb.client.std.ObjList; @@ -196,13 +196,21 @@ public ColumnBuffer getExistingColumn(CharSequence name, byte type) { *

    * Optimized for the common case where columns are accessed in the same * order every row: a sequential cursor avoids hash map lookups entirely. + *

    + * Returns {@code null} when the column has already been written in the current + * (uncommitted) row. Callers must treat a {@code null} return as "duplicate + * column in this row — skip the write", matching the ILP first-value-wins + * semantics. The check is a single field comparison on the hot path and has + * no measurable cost. */ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean nullable) { ColumnBuffer existing = lookupColumn(name, type); if (existing != null) { - return existing; + // col.size > rowCount means this column already received a value + // for the in-progress row. Silently ignore the duplicate (first + // value wins, same as the ILP server behaviour). + return existing.size <= rowCount ? existing : null; } - return createColumn(name, type, nullable); } From 7c7c4a32a892eba0866d13a11cb0b8ce56d20467 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 19 Mar 2026 10:04:50 +0100 Subject: [PATCH 212/230] Fix non-ASCII column name case insensitivity --- ...bstractLowerCaseAsciiCharSequenceHashSet.java | 6 +++--- .../main/java/io/questdb/client/std/Chars.java | 16 ++++++++++++++++ .../LowerCaseAsciiCharSequenceIntHashMap.java | 4 ++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java index 91bbc5f..4424999 100644 --- a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java +++ b/core/src/main/java/io/questdb/client/std/AbstractLowerCaseAsciiCharSequenceHashSet.java @@ -54,13 +54,13 @@ public void clear() { } public int keyIndex(CharSequence key) { - int index = Chars.lowerCaseAsciiHashCode(key) & mask; + int index = Chars.lowerCaseHashCode(key) & mask; if (keys[index] == noEntryKey) { return index; } - if (Chars.equalsLowerCaseAscii(key, keys[index])) { + if (Chars.equalsIgnoreCase(key, keys[index])) { return -index - 1; } @@ -77,7 +77,7 @@ private int probe(CharSequence key, int index) { if (keys[index] == noEntryKey) { return index; } - if (Chars.equalsLowerCaseAscii(key, keys[index])) { + if (Chars.equalsIgnoreCase(key, keys[index])) { return -index - 1; } } while (true); diff --git a/core/src/main/java/io/questdb/client/std/Chars.java b/core/src/main/java/io/questdb/client/std/Chars.java index f75c54d..b3ded7f 100644 --- a/core/src/main/java/io/questdb/client/std/Chars.java +++ b/core/src/main/java/io/questdb/client/std/Chars.java @@ -364,6 +364,22 @@ public static boolean startsWith(CharSequence _this, char c) { return _this.length() > 0 && _this.charAt(0) == c; } + public static String toLowerCase(@Nullable CharSequence value) { + if (value == null) { + return null; + } + final int len = value.length(); + if (len == 0) { + return ""; + } + + final Utf16Sink b = Misc.getThreadLocalSink(); + for (int i = 0; i < len; i++) { + b.put(Character.toLowerCase(value.charAt(i))); + } + return b.toString(); + } + public static String toLowerCaseAscii(@Nullable CharSequence value) { if (value == null) { return null; diff --git a/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java index e833b38..ab70edb 100644 --- a/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java +++ b/core/src/main/java/io/questdb/client/std/LowerCaseAsciiCharSequenceIntHashMap.java @@ -65,14 +65,14 @@ public boolean putAt(int index, CharSequence key, int value) { values[-index - 1] = value; return false; } - putAt0(index, Chars.toLowerCaseAscii(key), value); + putAt0(index, Chars.toLowerCase(key), value); return true; } public void putIfAbsent(CharSequence key, int value) { int index = keyIndex(key); if (index > -1) { - putAt0(index, Chars.toLowerCaseAscii(key), value); + putAt0(index, Chars.toLowerCase(key), value); } } From 5b1070b73027e2d4f20f27c0c3676918fc1578c6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 19 Mar 2026 15:28:25 +0100 Subject: [PATCH 213/230] Fix caught exception type in rollback test flushSync() wraps the NullPointerException from sendBinary() on a null client in a LineSenderException via its catch (Throwable) block. The test was catching the unwrapped NullPointerException, which never arrived, causing the test to fail with an unhandled LineSenderException. Co-Authored-By: Claude Opus 4.6 --- .../test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java index 90897c8..eeccce0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.InFlightWindow; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.test.AbstractTest; @@ -65,9 +66,9 @@ public void testSyncFlushFailureDoesNotAdvanceMaxSentSymbolId() throws Exception // because client is null (we never actually connected) try { sender.flush(); - Assert.fail("Expected NullPointerException from null client"); - } catch (NullPointerException expected) { - // sendBinary() on null client + Assert.fail("Expected LineSenderException from null client"); + } catch (LineSenderException expected) { + // sendBinary() on null client, wrapped by flushSync() } // The fix: maxSentSymbolId must remain -1 because the send failed. From d46895e4a4b2bfce43c88e5e3b70f637e5b70d4d Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 11:13:46 +0100 Subject: [PATCH 214/230] do not account for pending bytes when auto-flushing bytes is disabled --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 10 +++++++--- .../cutlass/line/tcp/v4/QwpAllocationTestClient.java | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 5df7b45..6a9f1c7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1298,9 +1298,13 @@ private void sealAndSwapBuffer() { */ private void sendRow() { ensureConnected(); - long bytesBefore = currentTableBuffer.getBufferedBytes(); - currentTableBuffer.nextRow(); - pendingBytes += currentTableBuffer.getBufferedBytes() - bytesBefore; + if (autoFlushBytes > 0) { + long bytesBefore = currentTableBuffer.getBufferedBytes(); + currentTableBuffer.nextRow(); + pendingBytes += currentTableBuffer.getBufferedBytes() - bytesBefore; + } else { + currentTableBuffer.nextRow(); + } // Both modes: accumulate rows, don't encode yet if (pendingRowCount == 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 17a3d55..10694c1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -209,7 +209,7 @@ private static Sender createSender( .address(host) .port(port); if (batchSize > 0) wsBuilder.autoFlushRows(batchSize); - if (flushBytes > 0) wsBuilder.autoFlushBytes(flushBytes); + if (flushBytes >= 0) wsBuilder.autoFlushBytes(flushBytes); if (flushIntervalMs > 0) wsBuilder.autoFlushIntervalMillis((int) flushIntervalMs); if (inFlightWindow > 0) wsBuilder.inFlightWindowSize(inFlightWindow); return wsBuilder.build(); From 53609ce0db6a84f94dc2bf122c00df1f2ef1de76 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 20 Mar 2026 11:37:37 +0100 Subject: [PATCH 215/230] Avoid String allocation in column name validation --- .../questdb/client/cutlass/qwp/client/QwpWebSocketSender.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 6a9f1c7..09a2e9d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -965,7 +965,7 @@ private void checkTableSelected() { } } - private String checkedColumnName(CharSequence name) { + private CharSequence checkedColumnName(CharSequence name) { if (name == null || !TableUtils.isValidColumnName(name, DEFAULT_MAX_NAME_LENGTH)) { if (name == null || name.length() == 0) { throw new LineSenderException("column name cannot be empty"); @@ -975,7 +975,7 @@ private String checkedColumnName(CharSequence name) { } throw new LineSenderException("column name contains illegal characters: " + name); } - return name.toString(); + return name; } /** From 005357947af9cf08a403c67cf0664ea8e5d96298 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 20 Mar 2026 11:37:56 +0100 Subject: [PATCH 216/230] Recreate table in benchmark --- .../line/tcp/v4/QwpAllocationTestClient.java | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 10694c1..025064a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -27,7 +27,11 @@ import io.questdb.client.Sender; import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.time.temporal.ChronoUnit; +import java.util.Properties; import java.util.concurrent.TimeUnit; /** @@ -95,6 +99,7 @@ public class QwpAllocationTestClient { "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ", "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA" }; + private static final String TABLE_NAME = "ilp_alloc_test"; public static void main(String[] args) { // Parse command-line options @@ -173,6 +178,7 @@ public static void main(String[] args) { System.out.println(); try { + recreateTable(host); runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs, inFlightWindow, maxDatagramSize, warmupRows, reportInterval, targetThroughput); } catch (Exception e) { @@ -286,6 +292,35 @@ private static void printUsage() { System.out.println(" QwpAllocationTestClient --protocol=ilp-tcp --rows=100000 --no-warmup"); } + private static void recreateTable(String host) throws Exception { + Properties properties = new Properties(); + properties.setProperty("user", "admin"); + properties.setProperty("password", "quest"); + properties.setProperty("sslmode", "disable"); + String url = "jdbc:postgresql://" + host + ":8812/qdb"; + try (Connection conn = DriverManager.getConnection(url, properties); + Statement st = conn.createStatement() + ) { + st.execute("DROP TABLE IF EXISTS " + TABLE_NAME); + st.execute("CREATE TABLE " + TABLE_NAME + " (" + + " timestamp TIMESTAMP," + + " exchange SYMBOL," + + " currency SYMBOL," + + " trade_id LONG," + + " volume LONG," + + " price DOUBLE," + + " bid DOUBLE," + + " ask DOUBLE," + + " sequence LONG," + + " spread DOUBLE," + + " venue VARCHAR," + + " is_buy BOOLEAN," + + " event_time TIMESTAMP" + + ") TIMESTAMP(timestamp) PARTITION BY DAY WAL"); + } + System.out.println("Recreated table " + TABLE_NAME); + } + private static void runTest(String protocol, String host, int port, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, int inFlightWindow, int maxDatagramSize, @@ -345,9 +380,9 @@ private static void runTest(String protocol, String host, int port, int totalRow long now = System.nanoTime(); long elapsedSinceReport = now - lastReportTime; int rowsSinceReport = (i + 1) - lastReportRows; - double rowsPerSec = rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0); + int rowsPerSec = (int) (rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0)); - System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %.0f rows/sec%n", + System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %,d rows/sec%n", i + 1, totalRows, (i + 1) * 100.0 / totalRows, rowsPerSec); @@ -386,7 +421,7 @@ private static void sendRow(Sender sender, int rowIndex) { long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros long timestamp = baseTimestamp + (rowIndex * 1000L) + (rowIndex % 100); - sender.table("ilp_alloc_test") + sender.table(TABLE_NAME) // Symbol columns .symbol("exchange", SYMBOLS[rowIndex % SYMBOLS.length]) .symbol("currency", rowIndex % 2 == 0 ? "USD" : "EUR") From 96021be47d5244f9f39687337bcb5b207d0a62ae Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 20 Mar 2026 12:39:51 +0100 Subject: [PATCH 217/230] Validate only new column names --- .../qwp/client/QwpWebSocketSender.java | 75 ++++++++----------- .../cutlass/qwp/protocol/QwpTableBuffer.java | 18 +++++ 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 09a2e9d..6447e84 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -93,9 +93,9 @@ public class QwpWebSocketSender implements Sender { public static final int DEFAULT_AUTO_FLUSH_ROWS = 1_000; public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 128; private static final int DEFAULT_BUFFER_SIZE = 8192; - private static final int DEFAULT_MAX_NAME_LENGTH = 127; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); + private static final int MAX_TABLE_NAME_LENGTH = 127; private static final String WRITE_PATH = "/write/v4"; private final AckFrameHandler ackHandler = new AckFrameHandler(this); private final WebSocketResponse ackResponse = new WebSocketResponse(); @@ -342,7 +342,7 @@ public void atNow() { public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_BOOLEAN, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_BOOLEAN, false); if (col != null) { col.addBoolean(value); } @@ -364,7 +364,7 @@ public DirectByteSlice bufferView() { public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_BYTE, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_BYTE, false); if (col != null) { col.addByte(value); } @@ -391,7 +391,7 @@ public void cancelRow() { public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_CHAR, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_CHAR, false); if (col != null) { col.addShort((short) value); } @@ -474,7 +474,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL64, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL64, true); if (col != null) { col.addDecimal64(value); } @@ -486,7 +486,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL128, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL128, true); if (col != null) { col.addDecimal128(value); } @@ -498,7 +498,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL256, true); if (col != null) { col.addDecimal256(value); } @@ -512,7 +512,7 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { checkTableSelected(); try { currentDecimal256.ofString(value); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DECIMAL256, true); if (col != null) { col.addDecimal256(currentDecimal256); } @@ -527,7 +527,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } @@ -539,7 +539,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } @@ -551,7 +551,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } @@ -563,7 +563,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(array); } @@ -574,7 +574,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_DOUBLE, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_DOUBLE, true); if (col != null) { col.addDouble(value); } @@ -591,7 +591,7 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_FLOAT, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_FLOAT, true); if (col != null) { col.addFloat(value); } @@ -703,7 +703,7 @@ public QwpTableBuffer getTableBuffer(String tableName) { public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_INT, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_INT, true); if (col != null) { col.addInt(value); } @@ -730,7 +730,7 @@ public boolean isGorillaEnabled() { public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_LONG256, true); if (col != null) { col.addLong256(l0, l1, l2, l3); } @@ -742,7 +742,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } @@ -754,7 +754,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } @@ -766,7 +766,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } @@ -778,7 +778,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(name), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(array); } @@ -789,7 +789,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_LONG, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_LONG, true); if (col != null) { col.addLong(value); } @@ -834,7 +834,7 @@ public void setGorillaEnabled(boolean enabled) { public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_SHORT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_SHORT, false); if (col != null) { col.addShort(value); } @@ -845,7 +845,7 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_STRING, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_STRING, true); if (col != null) { col.addString(value); } @@ -856,7 +856,7 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_SYMBOL, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_SYMBOL, true); if (col != null) { col.addSymbol(value); } @@ -889,13 +889,13 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkNotClosed(); checkTableSelected(); if (unit == ChronoUnit.NANOS) { - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP_NANOS, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP_NANOS, true); if (col != null) { col.addLong(value); } } else { long micros = toMicros(value, unit); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); if (col != null) { col.addLong(micros); } @@ -908,7 +908,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkNotClosed(); checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_TIMESTAMP, true); if (col != null) { col.addLong(micros); } @@ -926,7 +926,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(checkedColumnName(columnName), TYPE_UUID, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, TYPE_UUID, true); if (col != null) { col.addUuid(hi, lo); } @@ -965,19 +965,6 @@ private void checkTableSelected() { } } - private CharSequence checkedColumnName(CharSequence name) { - if (name == null || !TableUtils.isValidColumnName(name, DEFAULT_MAX_NAME_LENGTH)) { - if (name == null || name.length() == 0) { - throw new LineSenderException("column name cannot be empty"); - } - if (name.length() > DEFAULT_MAX_NAME_LENGTH) { - throw new LineSenderException("column name too long [maxLength=" + DEFAULT_MAX_NAME_LENGTH + "]"); - } - throw new LineSenderException("column name contains illegal characters: " + name); - } - return name; - } - /** * Ensures the active buffer is ready for writing (in FILLING state). * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. @@ -1365,12 +1352,12 @@ private long toMicros(long value, ChronoUnit unit) { } private void validateTableName(CharSequence name) { - if (name == null || !TableUtils.isValidTableName(name, DEFAULT_MAX_NAME_LENGTH)) { + if (name == null || !TableUtils.isValidTableName(name, MAX_TABLE_NAME_LENGTH)) { if (name == null || name.length() == 0) { throw new LineSenderException("table name cannot be empty"); } - if (name.length() > DEFAULT_MAX_NAME_LENGTH) { - throw new LineSenderException("table name too long [maxLength=" + DEFAULT_MAX_NAME_LENGTH + "]"); + if (name.length() > MAX_TABLE_NAME_LENGTH) { + throw new LineSenderException("table name too long [maxLength=" + MAX_TABLE_NAME_LENGTH + "]"); } throw new LineSenderException("table name contains illegal characters: " + name); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index d18ec82..991bc7c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cairo.TableUtils; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.cutlass.line.array.DoubleArray; @@ -59,6 +60,7 @@ */ public class QwpTableBuffer implements QuietCloseable { + private static final int MAX_COLUMN_NAME_LENGTH = 127; private final LowerCaseAsciiCharSequenceIntHashMap columnNameToIndex; private final ObjList columns; private final QwpWebSocketSender sender; @@ -343,6 +345,10 @@ private static void assertColumnType(CharSequence name, byte type, ColumnBuffer } private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) { + // empty name is the designated timestamp sentinel — skip validation + if (name.length() > 0) { + validateColumnName(name); + } ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); col.sender = sender; int index = columns.size(); @@ -415,6 +421,18 @@ private void rebuildColumnAccessStructures() { cachedColumnDefs = null; } + private static void validateColumnName(CharSequence name) { + if (name == null || !TableUtils.isValidColumnName(name, MAX_COLUMN_NAME_LENGTH)) { + if (name == null || name.length() == 0) { + throw new LineSenderException("column name cannot be empty"); + } + if (name.length() > MAX_COLUMN_NAME_LENGTH) { + throw new LineSenderException("column name too long [maxLength=" + MAX_COLUMN_NAME_LENGTH + "]"); + } + throw new LineSenderException("column name contains illegal characters: " + name); + } + } + /** * Returns the in-memory buffer element stride in bytes. This is the size used * to store each value in the client's off-heap {@link OffHeapAppendMemory} buffer. From 42ba7cf95937b038fc77f885ffd63d757f0da9d0 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 12:49:22 +0100 Subject: [PATCH 218/230] the fast path can be faster --- .../questdb/client/cutlass/qwp/client/QwpWebSocketSender.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 6447e84..06513a7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -866,12 +866,12 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { @Override public QwpWebSocketSender table(CharSequence tableName) { checkNotClosed(); - validateTableName(tableName); // Fast path: if table name matches current, skip hashmap lookup if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { return this; } // Table changed - invalidate cached column references + validateTableName(tableName); cachedTimestampColumn = null; cachedTimestampNanosColumn = null; currentTableName = tableName.toString(); From 45749d8c72e6650333e75b52475d42e3d5d45c31 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 13:34:07 +0100 Subject: [PATCH 219/230] --no-drop --- .../cutlass/line/tcp/v4/QwpAllocationTestClient.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 025064a..b73a358 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -115,6 +115,7 @@ public static void main(String[] args) { int warmupRows = DEFAULT_WARMUP_ROWS; int reportInterval = DEFAULT_REPORT_INTERVAL; int targetThroughput = DEFAULT_TARGET_THROUGHPUT; + boolean isDropTable = true; for (String arg : args) { if (arg.equals("--help") || arg.equals("-h")) { @@ -146,6 +147,8 @@ public static void main(String[] args) { targetThroughput = Integer.parseInt(arg.substring("--target-throughput=".length())); } else if (arg.equals("--no-warmup")) { warmupRows = 0; + } else if (arg.equals("--no-drop")) { + isDropTable = false; } else if (!arg.startsWith("--")) { // Legacy positional args: protocol [host] [port] [rows] protocol = arg.toLowerCase(); @@ -178,7 +181,11 @@ public static void main(String[] args) { System.out.println(); try { - recreateTable(host); + if (isDropTable) { + recreateTable(host); + } else { + System.out.println("Skipping table drop (--no-drop)"); + } runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs, inFlightWindow, maxDatagramSize, warmupRows, reportInterval, targetThroughput); } catch (Exception e) { @@ -277,6 +284,7 @@ private static void printUsage() { System.out.println(" --report=N Report progress every N rows (default: 1000000)"); System.out.println(" --target-throughput=N Target throughput in rows/sec (0 = unlimited, default: 0)"); System.out.println(" --no-warmup Skip warmup phase"); + System.out.println(" --no-drop Don't drop/recreate the table (for parallel clients)"); System.out.println(" --help Show this help"); System.out.println(); System.out.println("Protocols:"); From 12ca3bbf3db1d789e8b4f51d25ed31feb4d42ac9 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 20 Mar 2026 13:16:10 +0100 Subject: [PATCH 220/230] @NotNull annotation on isValidColumnName --- core/src/main/java/io/questdb/client/cairo/TableUtils.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cairo/TableUtils.java b/core/src/main/java/io/questdb/client/cairo/TableUtils.java index def4f02..f250e9f 100644 --- a/core/src/main/java/io/questdb/client/cairo/TableUtils.java +++ b/core/src/main/java/io/questdb/client/cairo/TableUtils.java @@ -24,8 +24,10 @@ package io.questdb.client.cairo; +import org.jetbrains.annotations.NotNull; + public final class TableUtils { - public static boolean isValidColumnName(CharSequence columnName, int fsFileNameLimit) { + public static boolean isValidColumnName(@NotNull CharSequence columnName, int fsFileNameLimit) { final int length = columnName.length(); if (length > fsFileNameLimit) { // Most file systems do not support file names longer than 255 bytes @@ -133,4 +135,4 @@ public static boolean isValidTableName(CharSequence tableName, int fsFileNameLim } return length > 0 && tableName.charAt(0) != ' ' && tableName.charAt(length - 1) != ' '; } -} \ No newline at end of file +} From c8e2fc2419ba493d6e90de37a21fec573cf511d6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 20 Mar 2026 13:44:04 +0100 Subject: [PATCH 221/230] Move column name validation into QwpTableBuffer QwpWebSocketSender.checkedColumnName() validated every column name on every row write, even though getOrCreateColumn() finds the column already exists in most cases. Move the validation into QwpTableBuffer.getOrCreateColumn() so it runs only when a new column is created. Add getOrCreateDesignatedTimestampColumn() for the designated timestamp, which uses an empty-string sentinel name and must bypass column name validation. Both QwpWebSocketSender and QwpUdpSender now use this dedicated method. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../cutlass/qwp/client/QwpUdpSender.java | 33 ++++++++++++----- .../qwp/client/QwpWebSocketSender.java | 4 +-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 35 ++++++++++--------- .../qwp/client/QwpWebSocketEncoderTest.java | 20 +++++------ 4 files changed, 54 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index aa3e8d1..65c3eb5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -723,6 +723,22 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, return col; } + private QwpTableBuffer.ColumnBuffer acquireDesignatedTimestampColumn(byte type) { + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn("", type); + if (col == null && currentTableBuffer.getRowCount() > 0) { + if (hasInProgressRow()) { + flushCommittedPrefixPreservingCurrentRow(); + } else { + flushCommittedRowsOfCurrentTable(); + } + col = currentTableBuffer.getExistingColumn("", type); + } + if (col == null) { + col = currentTableBuffer.getOrCreateDesignatedTimestampColumn(type); + } + return col; + } + private void appendDoubleArrayValue(QwpTableBuffer.ColumnBuffer column, Object value) { if (value instanceof double[]) { column.addDoubleArray((double[]) value); @@ -1216,16 +1232,15 @@ private void stageDecimal64ColumnValue(CharSequence name, Decimal64 value) { private void stageDesignatedTimestampValue(long value, boolean nanos) { QwpTableBuffer.ColumnBuffer col; - if (nanos) { - if (cachedTimestampNanosColumn == null) { - cachedTimestampNanosColumn = acquireColumn("", TYPE_TIMESTAMP_NANOS, true); - } - col = cachedTimestampNanosColumn; - } else { - if (cachedTimestampColumn == null) { - cachedTimestampColumn = acquireColumn("", TYPE_TIMESTAMP, true); + byte type = nanos ? TYPE_TIMESTAMP_NANOS : TYPE_TIMESTAMP; + col = nanos ? cachedTimestampNanosColumn : cachedTimestampColumn; + if (col == null) { + col = acquireDesignatedTimestampColumn(type); + if (nanos) { + cachedTimestampNanosColumn = col; + } else { + cachedTimestampColumn = col; } - col = cachedTimestampColumn; } beginColumnWrite(col, ""); col.addLong(value); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 06513a7..3506884 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -937,7 +937,7 @@ private void atMicros(long timestampMicros) { // Add designated timestamp column (empty name for designated timestamp) // Use cached reference to avoid hashmap lookup per row if (cachedTimestampColumn == null) { - cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + cachedTimestampColumn = currentTableBuffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); } cachedTimestampColumn.addLong(timestampMicros); sendRow(); @@ -947,7 +947,7 @@ private void atNanos(long timestampNanos) { // Add designated timestamp column (empty name for designated timestamp) // Use cached reference to avoid hashmap lookup per row if (cachedTimestampNanosColumn == null) { - cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); + cachedTimestampNanosColumn = currentTableBuffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP_NANOS); } cachedTimestampNanosColumn.addLong(timestampNanos); sendRow(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 991bc7c..adfaa76 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -206,6 +206,9 @@ public ColumnBuffer getExistingColumn(CharSequence name, byte type) { * no measurable cost. */ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean nullable) { + if (name == null || name.length() == 0) { + throw new LineSenderException("column name cannot be empty"); + } ColumnBuffer existing = lookupColumn(name, type); if (existing != null) { // col.size > rowCount means this column already received a value @@ -213,7 +216,21 @@ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean null // value wins, same as the ILP server behaviour). return existing.size <= rowCount ? existing : null; } - return createColumn(name, type, nullable); + if (TableUtils.isValidColumnName(name, MAX_COLUMN_NAME_LENGTH)) { + return createColumn(name, type, nullable); + } + throw new LineSenderException( + name.length() > MAX_COLUMN_NAME_LENGTH ? "column name too long [maxLength=" + MAX_COLUMN_NAME_LENGTH + "]" + : "column name contains illegal characters: " + name + ); + } + + public ColumnBuffer getOrCreateDesignatedTimestampColumn(byte type) { + ColumnBuffer existing = lookupColumn("", type); + if (existing != null) { + return existing; + } + return createColumn("", type, true); } /** @@ -345,10 +362,6 @@ private static void assertColumnType(CharSequence name, byte type, ColumnBuffer } private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) { - // empty name is the designated timestamp sentinel — skip validation - if (name.length() > 0) { - validateColumnName(name); - } ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); col.sender = sender; int index = columns.size(); @@ -421,18 +434,6 @@ private void rebuildColumnAccessStructures() { cachedColumnDefs = null; } - private static void validateColumnName(CharSequence name) { - if (name == null || !TableUtils.isValidColumnName(name, MAX_COLUMN_NAME_LENGTH)) { - if (name == null || name.length() == 0) { - throw new LineSenderException("column name cannot be empty"); - } - if (name.length() > MAX_COLUMN_NAME_LENGTH) { - throw new LineSenderException("column name too long [maxLength=" + MAX_COLUMN_NAME_LENGTH + "]"); - } - throw new LineSenderException("column name contains illegal characters: " + name); - } - } - /** * Returns the in-memory buffer element stride in bytes. This is the size used * to store each value in the client's off-heap {@link OffHeapAppendMemory} buffer. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 49adc6b..1cd013d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -135,7 +135,7 @@ public void testEncodeAllBasicTypesInOneRow() throws Exception { buffer.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(3.14159265); buffer.getOrCreateColumn("s", TYPE_STRING, true).addString("test"); buffer.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbol("AAPL"); - buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1000000L); + buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP).addLong(1000000L); buffer.nextRow(); @@ -380,7 +380,7 @@ public void testEncodeMixedColumnTypes() throws Exception { QwpTableBuffer.ColumnBuffer stringCol = buffer.getOrCreateColumn("message", TYPE_STRING, true); stringCol.addString("hello world"); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); tsCol.addLong(1000000L); buffer.nextRow(); @@ -407,7 +407,7 @@ public void testEncodeMixedColumnsMultipleRows() throws Exception { QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); doubleCol.addDouble(i * 1.5); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); tsCol.addLong(1000000L + i); buffer.nextRow(); @@ -435,7 +435,7 @@ public void testEncodeMultipleColumns() throws Exception { QwpTableBuffer.ColumnBuffer humCol = buffer.getOrCreateColumn("humidity", TYPE_LONG, false); humCol.addLong(65); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); tsCol.addLong(1000000L); buffer.nextRow(); @@ -489,7 +489,7 @@ public void testEncodeMultipleRows() throws Exception { QwpTableBuffer.ColumnBuffer valCol = buffer.getOrCreateColumn("value", TYPE_LONG, false); valCol.addLong(i); - QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); tsCol.addLong(1000000L + i); buffer.nextRow(); @@ -731,7 +731,7 @@ public void testEncodeSingleRowWithTimestamp() throws Exception { QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { // Add a timestamp column (designated timestamp uses empty name) - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); col.addLong(1000000L); // Micros buffer.nextRow(); @@ -1141,7 +1141,7 @@ public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() throws encoder.setGorillaEnabled(true); // Add multiple timestamps with constant delta (best compression) - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); for (int i = 0; i < 100; i++) { col.addLong(1000000000L + i * 1000L); buffer.nextRow(); @@ -1152,7 +1152,7 @@ public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() throws // Now encode without Gorilla encoder.setGorillaEnabled(false); buffer.reset(); - col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); for (int i = 0; i < 100; i++) { col.addLong(1000000000L + i * 1000L); buffer.nextRow(); @@ -1200,7 +1200,7 @@ public void testGorillaEncoding_singleTimestamp_usesUncompressed() throws Except encoder.setGorillaEnabled(true); // Single timestamp - should use uncompressed - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); col.addLong(1000000L); buffer.nextRow(); @@ -1218,7 +1218,7 @@ public void testGorillaEncoding_twoTimestamps_usesUncompressed() throws Exceptio encoder.setGorillaEnabled(true); // Only 2 timestamps - should use uncompressed (Gorilla needs 3+) - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP); col.addLong(1000000L); buffer.nextRow(); col.addLong(2000000L); From 52a5c1639ac714f6a40e6d4ccc9d73148d132b39 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 13:49:39 +0100 Subject: [PATCH 222/230] hello darkness (aka java logging APIs) my old friend... --- .../qwp/client/QwpWebSocketSender.java | 44 ++++++++++++++----- .../qwp/client/WebSocketSendQueue.java | 34 ++++++++++---- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3506884..59d68d5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -622,7 +622,9 @@ public void flush() { inFlightWindow.awaitEmpty(); } - LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); + if (LOG.isDebugEnabled()) { + LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); + } } else { // Sync mode (window=1): flush pending rows and wait for ACKs synchronously flushSync(); @@ -983,7 +985,9 @@ private void ensureActiveBufferReady() { // Buffer is in use (SEALED or SENDING) - wait for it // Use a while loop to handle spurious wakeups and race conditions with the latch while (activeBuffer.isInUse()) { - LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); if (!recycled) { throw new LineSenderException("Timeout waiting for active buffer to be recycled"); @@ -1064,7 +1068,9 @@ private void flushPendingRows() { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - LOG.debug("Flushing pending rows [count={}, tables={}]", pendingRowCount, tableBuffers.size()); + if (LOG.isDebugEnabled()) { + LOG.debug("Flushing pending rows [count={}, tables={}]", pendingRowCount, tableBuffers.size()); + } // Ensure activeBuffer is ready for writing // It might be in RECYCLED state if previous batch was sent but we didn't swap yet @@ -1090,7 +1096,9 @@ private void flushPendingRows() { long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); - LOG.debug("Encoding table [name={}, rows={}, maxSentSymbolId={}, batchMaxId={}, useSchemaRef={}]", tableName, rowCount, maxSentSymbolId, currentBatchMaxSymbolId, useSchemaRef); + if (LOG.isDebugEnabled()) { + LOG.debug("Encoding table [name={}, rows={}, maxSentSymbolId={}, batchMaxId={}, useSchemaRef={}]", tableName, rowCount, maxSentSymbolId, currentBatchMaxSymbolId, useSchemaRef); + } // Encode this table's rows with delta symbol dictionary int messageSize = encoder.encodeWithDeltaDict( @@ -1147,7 +1155,9 @@ private void flushSync() { cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); + if (LOG.isDebugEnabled()) { + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); + } // Encode all table buffers that have data into a single message ObjList keys = tableBuffers.keys(); @@ -1183,7 +1193,9 @@ private void flushSync() { long batchSequence = nextBatchSequence++; inFlightWindow.addInFlight(batchSequence); - LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), currentBatchMaxSymbolId, useSchemaRef); + if (LOG.isDebugEnabled()) { + LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), currentBatchMaxSymbolId, useSchemaRef); + } // Send over WebSocket and fail the in-flight entry if send throws, // so close() does not hang waiting for an ACK that will never arrive. @@ -1223,7 +1235,9 @@ private void flushSync() { pendingRowCount = 0; firstPendingRowTimeNanos = 0; - LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); + if (LOG.isDebugEnabled()) { + LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); + } } private long getPendingBytes() { @@ -1242,7 +1256,9 @@ private void sealAndSwapBuffer() { MicrobatchBuffer toSend = activeBuffer; toSend.seal(); - LOG.debug("Sealing buffer [id={}, rows={}, bytes={}]", toSend.getBatchId(), toSend.getRowCount(), toSend.getBufferPos()); + if (LOG.isDebugEnabled()) { + LOG.debug("Sealing buffer [id={}, rows={}, bytes={}]", toSend.getBatchId(), toSend.getRowCount(), toSend.getBufferPos()); + } // Swap to the other buffer activeBuffer = (activeBuffer == buffer0) ? buffer1 : buffer0; @@ -1250,17 +1266,23 @@ private void sealAndSwapBuffer() { // If the other buffer is still being sent, wait for it // Use a while loop to handle spurious wakeups and race conditions with the latch while (activeBuffer.isInUse()) { - LOG.debug("Waiting for buffer recycle [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for buffer recycle [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); if (!recycled) { throw new LineSenderException("Timeout waiting for buffer to be recycled"); } - LOG.debug("Buffer recycled [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + if (LOG.isDebugEnabled()) { + LOG.debug("Buffer recycled [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } } // Reset the new active buffer int stateBeforeReset = activeBuffer.getState(); - LOG.debug("Resetting buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(stateBeforeReset)); + if (LOG.isDebugEnabled()) { + LOG.debug("Resetting buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(stateBeforeReset)); + } activeBuffer.reset(); // Enqueue the sealed buffer for sending. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index e249351..9ffda54 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -265,7 +265,9 @@ public boolean enqueue(MicrobatchBuffer buffer) { } } } - LOG.debug("Enqueued batch [id={}, bytes={}, rows={}]", buffer.getBatchId(), buffer.getBufferPos(), buffer.getRowCount()); + if (LOG.isDebugEnabled()) { + LOG.debug("Enqueued batch [id={}, bytes={}, rows={}]", buffer.getBatchId(), buffer.getBufferPos(), buffer.getRowCount()); + } return true; } @@ -548,23 +550,33 @@ private void sendBatch(MicrobatchBuffer batch) { int bytes = batch.getBufferPos(); int rows = batch.getRowCount(); - LOG.debug("Sending batch [seq={}, bytes={}, rows={}, bufferId={}]", batchSequence, bytes, rows, batch.getBatchId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Sending batch [seq={}, bytes={}, rows={}, bufferId={}]", batchSequence, bytes, rows, batch.getBatchId()); + } // Add to in-flight window BEFORE sending (so we're ready for ACK) // Use non-blocking tryAddInFlight since we already checked window space in ioLoop if (inFlightWindow != null) { - LOG.debug("Adding to in-flight window [seq={}, inFlight={}, max={}]", batchSequence, inFlightWindow.getInFlightCount(), inFlightWindow.getMaxWindowSize()); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding to in-flight window [seq={}, inFlight={}, max={}]", batchSequence, inFlightWindow.getInFlightCount(), inFlightWindow.getMaxWindowSize()); + } if (!inFlightWindow.tryAddInFlight(batchSequence)) { // Should not happen since we checked hasWindowSpace before polling throw new LineSenderException("In-flight window unexpectedly full"); } - LOG.debug("Added to in-flight window [seq={}]", batchSequence); + if (LOG.isDebugEnabled()) { + LOG.debug("Added to in-flight window [seq={}]", batchSequence); + } } // Send over WebSocket - LOG.debug("Calling sendBinary [seq={}]", batchSequence); + if (LOG.isDebugEnabled()) { + LOG.debug("Calling sendBinary [seq={}]", batchSequence); + } client.sendBinary(batch.getBufferPtr(), bytes); - LOG.debug("sendBinary returned [seq={}]", batchSequence); + if (LOG.isDebugEnabled()) { + LOG.debug("sendBinary returned [seq={}]", batchSequence); + } // Update statistics totalBatchesSent.incrementAndGet(); @@ -573,7 +585,9 @@ private void sendBatch(MicrobatchBuffer batch) { // Transition state: SENDING -> RECYCLED batch.markRecycled(); - LOG.debug("Batch sent and recycled [seq={}, bufferId={}]", batchSequence, batch.getBatchId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Batch sent and recycled [seq={}, bufferId={}]", batchSequence, batch.getBatchId()); + } } /** @@ -639,8 +653,10 @@ public void onBinaryMessage(long payloadPtr, int payloadLen) { int acked = inFlightWindow.acknowledgeUpTo(sequence); if (acked > 0) { totalAcks.addAndGet(acked); - LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); - } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); + } + } else if (LOG.isDebugEnabled()) { LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); } } From 9b49eed9bfee112eba3437a95f62311f0a36b7d9 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 14:39:56 +0100 Subject: [PATCH 223/230] reduce allocation pressure --- .../cutlass/qwp/client/MicrobatchBuffer.java | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index dee78d2..24f10f5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -28,9 +28,9 @@ import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.LockSupport; /** * A buffer for accumulating ILP data into microbatches before sending. @@ -75,8 +75,7 @@ public class MicrobatchBuffer implements QuietCloseable { // Symbol tracking for delta encoding private int maxSymbolId = -1; // For waiting on recycle (user thread waits for I/O thread to finish) - // CountDownLatch is not resettable, so we create a new instance on reset() - private volatile CountDownLatch recycleLatch = new CountDownLatch(1); + private volatile Thread recycleWaiter; // Row tracking private int rowCount; // State machine @@ -137,10 +136,20 @@ public static String stateName(int state) { * Only the user thread should call this. */ public void awaitRecycled() { + final Thread current = Thread.currentThread(); + recycleWaiter = current; try { - recycleLatch.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + while (state != STATE_RECYCLED) { + LockSupport.park(this); + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + return; + } + } + } finally { + if (recycleWaiter == current) { + recycleWaiter = null; + } } } @@ -152,11 +161,31 @@ public void awaitRecycled() { * @return true if recycled, false if timeout elapsed */ public boolean awaitRecycled(long timeout, TimeUnit unit) { + if (state == STATE_RECYCLED) { + // fast-path + return true; + } + + final Thread current = Thread.currentThread(); + recycleWaiter = current; + final long deadlineNanos = System.nanoTime() + unit.toNanos(timeout); try { - return recycleLatch.await(timeout, unit); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; + while (state != STATE_RECYCLED) { + final long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0) { + return false; + } + LockSupport.parkNanos(this, remaining); + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + return false; + } + } + return true; + } finally { + if (recycleWaiter == current) { + recycleWaiter = null; + } } } @@ -332,7 +361,10 @@ public void markRecycled() { throw new IllegalStateException("Cannot mark recycled in state " + stateName(state)); } state = STATE_RECYCLED; - recycleLatch.countDown(); + Thread w = recycleWaiter; + if (w != null) { + LockSupport.unpark(w); + } } /** @@ -364,8 +396,8 @@ public void reset() { firstRowTimeNanos = 0; maxSymbolId = -1; batchId = nextBatchId.getAndIncrement(); + recycleWaiter = null; state = STATE_FILLING; - recycleLatch = new CountDownLatch(1); } /** From 8afd5f628e1ade6574d6e6e997ea0f3c27f1d184 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 14:44:38 +0100 Subject: [PATCH 224/230] dead code removed --- .../cutlass/qwp/client/MicrobatchBuffer.java | 54 -------------- .../qwp/client/QwpWebSocketSender.java | 1 - .../qwp/client/MicrobatchBufferTest.java | 73 +------------------ 3 files changed, 2 insertions(+), 126 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 24f10f5..aa19541 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -61,10 +61,7 @@ public class MicrobatchBuffer implements QuietCloseable { public static final int STATE_SEALED = 1; public static final int STATE_SENDING = 2; private static final AtomicLong nextBatchId = new AtomicLong(); - private final long maxAgeNanos; - private final int maxBytes; // Flush trigger thresholds - private final int maxRows; // Batch identification private long batchId; private int bufferCapacity; @@ -72,8 +69,6 @@ public class MicrobatchBuffer implements QuietCloseable { // Native memory buffer private long bufferPtr; private long firstRowTimeNanos; - // Symbol tracking for delta encoding - private int maxSymbolId = -1; // For waiting on recycle (user thread waits for I/O thread to finish) private volatile Thread recycleWaiter; // Row tracking @@ -98,9 +93,6 @@ public MicrobatchBuffer(int initialCapacity, int maxRows, int maxBytes, long max this.bufferPos = 0; this.rowCount = 0; this.firstRowTimeNanos = 0; - this.maxRows = maxRows; - this.maxBytes = maxBytes; - this.maxAgeNanos = maxAgeNanos; this.batchId = nextBatchId.getAndIncrement(); } @@ -288,24 +280,6 @@ public void incrementRowCount() { rowCount++; } - /** - * Checks if the age limit has been exceeded. - */ - public boolean isAgeLimitExceeded() { - if (maxAgeNanos <= 0 || rowCount == 0) { - return false; - } - long ageNanos = System.nanoTime() - firstRowTimeNanos; - return ageNanos >= maxAgeNanos; - } - - /** - * Checks if the byte size limit has been exceeded. - */ - public boolean isByteLimitExceeded() { - return maxBytes > 0 && bufferPos >= maxBytes; - } - /** * Returns true if the buffer is in FILLING state (available for writing). */ @@ -328,13 +302,6 @@ public boolean isRecycled() { return state == STATE_RECYCLED; } - /** - * Checks if the row count limit has been exceeded. - */ - public boolean isRowLimitExceeded() { - return maxRows > 0 && rowCount >= maxRows; - } - /** * Returns true if the buffer is in SEALED state (ready to send). */ @@ -394,7 +361,6 @@ public void reset() { bufferPos = 0; rowCount = 0; firstRowTimeNanos = 0; - maxSymbolId = -1; batchId = nextBatchId.getAndIncrement(); recycleWaiter = null; state = STATE_FILLING; @@ -445,26 +411,6 @@ public void setBufferPos(int pos) { this.bufferPos = pos; } - /** - * Sets the maximum symbol ID used in this batch. - * Used for delta symbol dictionary tracking. - */ - public void setMaxSymbolId(int maxSymbolId) { - this.maxSymbolId = maxSymbolId; - } - - /** - * Checks if the buffer should be flushed based on configured thresholds. - * - * @return true if any flush threshold is exceeded - */ - public boolean shouldFlush() { - if (!hasData()) { - return false; - } - return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded(); - } - @Override public String toString() { return "MicrobatchBuffer{" + diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 59d68d5..92e19d3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1116,7 +1116,6 @@ private void flushPendingRows() { activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); - activeBuffer.setMaxSymbolId(currentBatchMaxSymbolId); // Seal and enqueue for sending sealAndSwapBuffer(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java index 4d96e59..bda1b1f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -27,7 +27,9 @@ import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; + import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + import org.junit.Assert; import org.junit.Test; @@ -578,77 +580,6 @@ public void testSetBufferPosOutOfBounds() throws Exception { }); } - @Test - public void testShouldFlushAgeLimit() throws Exception { - assertMemoryLeak(() -> { - // 50ms timeout - try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 0, 50_000_000L)) { - buffer.writeByte((byte) 1); - buffer.incrementRowCount(); - Assert.assertFalse(buffer.shouldFlush()); - - Thread.sleep(60); - - Assert.assertTrue(buffer.shouldFlush()); - Assert.assertTrue(buffer.isAgeLimitExceeded()); - } - }); - } - - @Test - public void testShouldFlushByteLimit() throws Exception { - assertMemoryLeak(() -> { - try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 10, 0)) { - for (int i = 0; i < 9; i++) { - buffer.writeByte((byte) i); - buffer.incrementRowCount(); - Assert.assertFalse(buffer.shouldFlush()); - } - buffer.writeByte((byte) 9); - buffer.incrementRowCount(); - Assert.assertTrue(buffer.shouldFlush()); - Assert.assertTrue(buffer.isByteLimitExceeded()); - } - }); - } - - @Test - public void testShouldFlushEmptyBuffer() throws Exception { - assertMemoryLeak(() -> { - try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 1, 1, 1)) { - Assert.assertFalse(buffer.shouldFlush()); // Empty buffer never flushes - } - }); - } - - @Test - public void testShouldFlushRowLimit() throws Exception { - assertMemoryLeak(() -> { - try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 5, 0, 0)) { - for (int i = 0; i < 4; i++) { - buffer.writeByte((byte) i); - buffer.incrementRowCount(); - Assert.assertFalse(buffer.shouldFlush()); - } - buffer.writeByte((byte) 4); - buffer.incrementRowCount(); - Assert.assertTrue(buffer.shouldFlush()); - Assert.assertTrue(buffer.isRowLimitExceeded()); - } - }); - } - - @Test - public void testShouldFlushWithNoThresholds() throws Exception { - assertMemoryLeak(() -> { - try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { - buffer.writeByte((byte) 1); - buffer.incrementRowCount(); - Assert.assertFalse(buffer.shouldFlush()); // No thresholds set - } - }); - } - @Test public void testStateName() { Assert.assertEquals("FILLING", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_FILLING)); From aa557670b975d058d170a5862f0207e7cfe5d2f0 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Fri, 20 Mar 2026 16:15:39 +0100 Subject: [PATCH 225/230] Renaem nullable -> use/hasNullBitmap "Nullable" is a confusing concept due to QuestDB null sentinels, which means a "non-nullable" column could still hold nulls via sentinel values. --- .../cutlass/qwp/client/QwpColumnWriter.java | 2 +- .../cutlass/qwp/client/QwpUdpSender.java | 12 +++--- .../cutlass/qwp/protocol/QwpColumnDef.java | 38 +++++++++---------- .../cutlass/qwp/protocol/QwpConstants.java | 4 +- .../cutlass/qwp/protocol/QwpSchemaHash.java | 2 +- .../cutlass/qwp/protocol/QwpTableBuffer.java | 8 ++-- .../qwp/client/DeltaSymbolDictionaryTest.java | 2 +- .../cutlass/qwp/client/QwpUdpSenderTest.java | 2 +- .../qwp/protocol/QwpColumnDefTest.java | 2 +- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index 2a127fa..088e8e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -58,7 +58,7 @@ private void encodeColumn( ) { long dataAddr = col.getDataAddress(); - if (colDef.isNullable()) { + if (colDef.hasNullBitmap()) { writeNullBitmap(col, rowCount); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index 65c3eb5..181d4b2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -703,7 +703,7 @@ private static int utf8Length(CharSequence s) { return len; } - private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean nullable) { + private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, boolean useNullBitmap) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getExistingColumn(name, type); if (col == null && currentTableBuffer.getRowCount() > 0) { // schema change while having some rows accumulated -> we flush committed rows of the current table @@ -718,7 +718,7 @@ private QwpTableBuffer.ColumnBuffer acquireColumn(CharSequence name, byte type, } if (col == null) { - col = currentTableBuffer.getOrCreateColumn(name, type, nullable); + col = currentTableBuffer.getOrCreateColumn(name, type, useNullBitmap); } return col; } @@ -1043,7 +1043,7 @@ private long estimateCurrentDatagramSizeWithInProgressRow(int targetRows) { for (int i = 0; i < inProgressColumnCount; i++) { InProgressColumnState state = inProgressColumns[i]; estimate += state.payloadEstimateDelta; - if (state.nullable) { + if (state.useNullBitmap) { estimate += bitmapBytes(targetRows) - bitmapBytes(state.sizeBefore); } } @@ -1343,7 +1343,7 @@ private static final class InProgressColumnState { private int arrayDataOffsetBefore; private int arrayShapeOffsetBefore; private QwpTableBuffer.ColumnBuffer column; - private boolean nullable; + private boolean useNullBitmap; private long payloadEstimateDelta; private int sizeBefore; private long stringDataSizeBefore; @@ -1356,13 +1356,13 @@ void captureAfterWrite() { void clear() { column = null; - nullable = false; + useNullBitmap = false; payloadEstimateDelta = 0; } void of(QwpTableBuffer.ColumnBuffer column) { this.column = column; - this.nullable = column.usesNullBitmap(); + this.useNullBitmap = column.usesNullBitmap(); this.payloadEstimateDelta = 0; this.sizeBefore = column.getSize(); this.valueCountBefore = column.getValueCount(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index 88efb38..0f887e4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -34,33 +34,33 @@ */ public final class QwpColumnDef { private final String name; - private final boolean nullable; + private final boolean hasNullBitmap; private final byte typeCode; /** * Creates a column definition. * * @param name the column name (UTF-8) - * @param typeCode the QWP v1 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable) + * @param typeCode the QWP v1 type code (0x01-0x0F, optionally OR'd with 0x80 for null bitmap) */ public QwpColumnDef(String name, byte typeCode) { this.name = name; - // Extract nullable flag (high bit) and base type - this.nullable = (typeCode & 0x80) != 0; + // Extract null bitmap flag (high bit) and base type + this.hasNullBitmap = (typeCode & 0x80) != 0; this.typeCode = (byte) (typeCode & 0x7F); } /** - * Creates a column definition with explicit nullable flag. + * Creates a column definition with explicit null bitmap flag. * - * @param name the column name - * @param typeCode the base type code (0x01-0x0F) - * @param nullable whether the column is nullable + * @param name the column name + * @param typeCode the base type code (0x01-0x0F) + * @param hasNullBitmap whether the column has a null bitmap */ - public QwpColumnDef(String name, byte typeCode, boolean nullable) { + public QwpColumnDef(String name, byte typeCode, boolean hasNullBitmap) { this.name = name; this.typeCode = (byte) (typeCode & 0x7F); - this.nullable = nullable; + this.hasNullBitmap = hasNullBitmap; } @Override @@ -69,7 +69,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; QwpColumnDef that = (QwpColumnDef) o; return typeCode == that.typeCode && - nullable == that.nullable && + hasNullBitmap == that.hasNullBitmap && name.equals(that.name); } @@ -81,7 +81,7 @@ public String getName() { } /** - * Gets the base type code (without nullable flag). + * Gets the base type code (without null bitmap flag). * * @return type code 0x01-0x0F */ @@ -97,34 +97,34 @@ public String getTypeName() { } /** - * Gets the wire type code (with nullable flag if applicable). + * Gets the wire type code (with null bitmap flag if applicable). * * @return type code as sent on wire */ public byte getWireTypeCode() { - return nullable ? (byte) (typeCode | 0x80) : typeCode; + return hasNullBitmap ? (byte) (typeCode | 0x80) : typeCode; } @Override public int hashCode() { int result = name.hashCode(); result = 31 * result + typeCode; - result = 31 * result + (nullable ? 1 : 0); + result = 31 * result + (hasNullBitmap ? 1 : 0); return result; } /** - * Returns true if this column is nullable. + * Returns true if this column has a null bitmap. */ - public boolean isNullable() { - return nullable; + public boolean hasNullBitmap() { + return hasNullBitmap; } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(name).append(':').append(getTypeName()); - if (nullable) { + if (hasNullBitmap) { sb.append('?'); } return sb.toString(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index fef72eb..d0f31ce 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -309,7 +309,7 @@ public static int getFixedTypeSize(byte typeCode) { */ public static String getTypeName(byte typeCode) { int code = typeCode & TYPE_MASK; - boolean nullable = (typeCode & TYPE_NULLABLE_FLAG) != 0; + boolean hasNullBitmap = (typeCode & TYPE_NULLABLE_FLAG) != 0; String name; switch (code) { case TYPE_BOOLEAN: @@ -382,7 +382,7 @@ public static String getTypeName(byte typeCode) { name = "UNKNOWN(" + code + ")"; break; } - return nullable ? name + "?" : name; + return hasNullBitmap ? name + "?" : name; } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 43d8675..d5decc4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -152,7 +152,7 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList MAX_COLUMN_NAME_LENGTH ? "column name too long [maxLength=" + MAX_COLUMN_NAME_LENGTH + "]" @@ -361,8 +361,8 @@ private static void assertColumnType(CharSequence name, byte type, ColumnBuffer } } - private ColumnBuffer createColumn(CharSequence name, byte type, boolean nullable) { - ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, nullable); + private ColumnBuffer createColumn(CharSequence name, byte type, boolean useNullBitmap) { + ColumnBuffer col = new ColumnBuffer(Chars.toString(name), type, useNullBitmap); col.sender = sender; int index = columns.size(); col.index = index; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java index 9079dbb..7dbd212 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java @@ -184,7 +184,7 @@ public void testEdgeCase_nullSymbolValues() throws Exception { GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); try (QwpTableBuffer batch = new QwpTableBuffer("test")) { - QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, true); // nullable + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, true); // useNullBitmap int aaplId = globalDict.getOrAddSymbol("AAPL"); col.addSymbolWithGlobalId("AAPL", aaplId); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 68ba573..5a889b2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -2164,7 +2164,7 @@ private void decodeBooleans(Object[] values, boolean[] nulls, int valueCount) { } private ColumnValues decodeColumn(QwpColumnDef def, int rowCount) { - boolean[] nulls = def.isNullable() ? reader.readNullBitmap(rowCount) : new boolean[rowCount]; + boolean[] nulls = def.hasNullBitmap() ? reader.readNullBitmap(rowCount) : new boolean[rowCount]; int valueCount = rowCount - countNulls(nulls); Object[] values = new Object[rowCount]; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java index b942dc3..7859e0f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java @@ -80,7 +80,7 @@ public void testValidateNullableCharType() { byte nullableChar = (byte) (QwpConstants.TYPE_CHAR | QwpConstants.TYPE_NULLABLE_FLAG); QwpColumnDef col = new QwpColumnDef("ch", nullableChar); col.validate(); - assertTrue(col.isNullable()); + assertTrue(col.hasNullBitmap()); assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode()); } From 970d71716f6aefa48cf1dcf904fc5f8b38e17265 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 17:11:35 +0100 Subject: [PATCH 226/230] implement support for storing global-symbol-IDs-only in QwpTableBuffer this avoid extra work when just global dictionary is needed we need to keep the per-column dictionaries due to UDP. for now. --- .../qwp/client/QwpWebSocketSender.java | 8 +- .../cutlass/qwp/protocol/QwpTableBuffer.java | 43 +++++++++-- .../qwp/client/QwpWebSocketEncoderTest.java | 75 +++++++++++++++++++ .../qwp/protocol/QwpTableBufferTest.java | 48 ++++++++++++ 4 files changed, 163 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 92e19d3..8ffc8dd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1344,10 +1344,10 @@ private boolean shouldAutoFlush() { if (autoFlushBytes > 0 && getPendingBytes() >= autoFlushBytes) { return true; } - if (autoFlushIntervalNanos > 0) { - long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; - return ageNanos >= autoFlushIntervalNanos; - } +// if (autoFlushIntervalNanos > 0) { +// long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; +// return ageNanos >= autoFlushIntervalNanos; +// } return false; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index adfaa76..59dc5c4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -583,7 +583,7 @@ public static class ColumnBuffer implements QuietCloseable { private byte[] arrayDims; private int arrayShapeOffset; private int[] arrayShapes; - // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only) + // Optional auxiliary buffer used by symbol encoders that need sideband IDs. private OffHeapAppendMemory auxBuffer; // Off-heap data buffer for fixed-width types private OffHeapAppendMemory dataBuffer; @@ -605,6 +605,7 @@ public static class ColumnBuffer implements QuietCloseable { // Off-heap storage for string/varchar column data private OffHeapAppendMemory stringOffsets; // Symbol specific (dictionary stays on-heap) + private boolean storeGlobalSymbolIdsOnly; private CharSequenceIntHashMap symbolDict; private ObjList symbolList; private StringSink symbolLookupSink; @@ -1101,6 +1102,9 @@ public void addSymbol(CharSequence value) { return; } ensureNullBitmapCapacity(); + if (storeGlobalSymbolIdsOnly) { + throw new LineSenderException("column '" + name + "' cannot mix global symbol IDs with local symbol dictionary values"); + } int idx = getOrAddLocalSymbol(value); dataBuffer.putInt(idx); valueCount++; @@ -1129,6 +1133,9 @@ public void addSymbolUtf8(long ptr, int len) { return; } ensureNullBitmapCapacity(); + if (storeGlobalSymbolIdsOnly) { + throw new LineSenderException("column '" + name + "' cannot mix global symbol IDs with local symbol dictionary values"); + } int idx = getOrAddLocalSymbol(lookupSink); dataBuffer.putInt(idx); valueCount++; @@ -1141,13 +1148,24 @@ public void addSymbolWithGlobalId(CharSequence value, int globalId) { return; } ensureNullBitmapCapacity(); - int localIdx = getOrAddLocalSymbol(value); - dataBuffer.putInt(localIdx); - - if (auxBuffer == null) { - auxBuffer = new OffHeapAppendMemory(64); + if (!storeGlobalSymbolIdsOnly) { + if (symbolList != null && symbolList.size() > 0) { + int localIdx = getOrAddLocalSymbol(value); + dataBuffer.putInt(localIdx); + if (auxBuffer == null) { + auxBuffer = new OffHeapAppendMemory(64); + } + auxBuffer.putInt(globalId); + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; + } + valueCount++; + size++; + return; + } + storeGlobalSymbolIdsOnly = true; } - auxBuffer.putInt(globalId); + dataBuffer.putInt(globalId); if (globalId > maxGlobalSymbolId) { maxGlobalSymbolId = globalId; @@ -1312,6 +1330,9 @@ public String[] getSymbolDictionary() { } public int getSymbolDictionarySize() { + if (storeGlobalSymbolIdsOnly) { + return 0; + } return symbolList != null ? symbolList.size() : 0; } @@ -1364,6 +1385,7 @@ public void reset() { symbolDict.clear(); symbolList.clear(); } + storeGlobalSymbolIdsOnly = false; maxGlobalSymbolId = -1; arrayShapeOffset = 0; arrayDataOffset = 0; @@ -1480,6 +1502,7 @@ public void truncateTo(int newSize) { decimalScale = -1; geohashPrecision = -1; maxGlobalSymbolId = -1; + storeGlobalSymbolIdsOnly = false; if (symbolDict != null) { symbolDict.clear(); symbolList.clear(); @@ -1729,6 +1752,7 @@ private void resetEmptyMetadata() { decimalScale = -1; geohashPrecision = -1; maxGlobalSymbolId = -1; + storeGlobalSymbolIdsOnly = false; if (symbolDict != null) { symbolDict.clear(); symbolList.clear(); @@ -1798,6 +1822,11 @@ private void retainStringValue(int valueIndex) { private void retainSymbolValue(int valueIndex) { retainFixedWidthValue(valueIndex); + if (storeGlobalSymbolIdsOnly) { + maxGlobalSymbolId = Unsafe.getUnsafe().getInt(dataBuffer.pageAddress()); + return; + } + int localIndex = Unsafe.getUnsafe().getInt(dataBuffer.pageAddress()); String symbol = symbolList.get(localIndex); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 1cd013d..a118b0e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -32,6 +32,8 @@ import org.junit.Assert; import org.junit.Test; +import java.nio.charset.StandardCharsets; + import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -961,6 +963,43 @@ public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() throws Exceptio }); } + @Test + public void testEncodeWithDeltaDict_readsGlobalIdsFromDataBuffer() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + for (int i = 0; i < 8; i++) { + globalDict.getOrAddSymbol("SYM_" + i); + } + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("SYM_5", 5); + buffer.nextRow(); + col.addSymbolWithGlobalId("SYM_7", 7); + buffer.nextRow(); + + Assert.assertEquals(0, col.getAuxDataAddress()); + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, 7, 7, false); + Assert.assertTrue(size > 12); + + Cursor cursor = new Cursor(encoder.getBuffer().getBufferPtr() + HEADER_SIZE); + Assert.assertEquals(8, cursor.readVarint()); + Assert.assertEquals(0, cursor.readVarint()); + + Assert.assertEquals("test_table", cursor.readString()); + Assert.assertEquals(2, cursor.readVarint()); + Assert.assertEquals(1, cursor.readVarint()); + Assert.assertEquals(SCHEMA_MODE_FULL, cursor.readByte()); + Assert.assertEquals("ticker", cursor.readString()); + Assert.assertEquals(TYPE_SYMBOL, cursor.readByte()); + Assert.assertEquals(5, cursor.readVarint()); + Assert.assertEquals(7, cursor.readVarint()); + } + }); + } + // ==================== SCHEMA REFERENCE TESTS ==================== @Test @@ -1359,4 +1398,40 @@ public void testReset() throws Exception { } }); } + + private static final class Cursor { + private long address; + + private Cursor(long address) { + this.address = address; + } + + private byte readByte() { + return Unsafe.getUnsafe().getByte(address++); + } + + private String readString() { + int len = readVarint(); + byte[] bytes = new byte[len]; + for (int i = 0; i < len; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + String value = new String(bytes, StandardCharsets.UTF_8); + address += len; + return value; + } + + private int readVarint() { + int value = 0; + int shift = 0; + while (true) { + int b = Unsafe.getUnsafe().getByte(address++) & 0xff; + value |= (b & 0x7f) << shift; + if ((b & 0x80) == 0) { + return value; + } + shift += 7; + } + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 98e5ddb..24a3cd8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -328,6 +328,29 @@ public void testAddSymbolUtf8ReusesExistingDictionaryEntry() throws Exception { }); } + @Test + public void testAddSymbolWithGlobalIdStoresOnlyGlobalIds() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, true); + col.addSymbolWithGlobalId("alpha", 7); + table.nextRow(); + col.addSymbolWithGlobalId("beta", 11); + table.nextRow(); + + assertEquals(2, col.getSize()); + assertEquals(2, col.getValueCount()); + assertEquals(0, col.getSymbolDictionarySize()); + assertEquals(0, col.getAuxDataAddress()); + assertEquals(11, col.getMaxGlobalSymbolId()); + + long dataAddress = col.getDataAddress(); + assertEquals(7, Unsafe.getUnsafe().getInt(dataAddress)); + assertEquals(11, Unsafe.getUnsafe().getInt(dataAddress + Integer.BYTES)); + } + }); + } + @Test public void testCancelRowResetsDecimalScaleOnLateAddedColumn() throws Exception { assertMemoryLeak(() -> { @@ -406,6 +429,31 @@ public void testCancelRowResetsSymbolDictOnLateAddedColumn() throws Exception { }); } + @Test + public void testCancelRowRetainsGlobalSymbolIdWithoutLocalDictionary() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(0); + table.nextRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + QwpTableBuffer.ColumnBuffer colS = table.getOrCreateColumn("s", QwpConstants.TYPE_SYMBOL, true); + colS.addSymbolWithGlobalId("stale", 4); + table.cancelCurrentRow(); + + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(1); + colS.addSymbolWithGlobalId("fresh", 9); + table.nextRow(); + + assertEquals(2, colS.getSize()); + assertEquals(1, colS.getValueCount()); + assertEquals(0, colS.getSymbolDictionarySize()); + assertEquals(9, colS.getMaxGlobalSymbolId()); + assertEquals(9, Unsafe.getUnsafe().getInt(colS.getDataAddress())); + } + }); + } + @Test public void testCancelRowRewindsDoubleArrayOffsets() throws Exception { assertMemoryLeak(() -> { From 3fa1a0797fac539194c3011b8369bc3211fe20c5 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 20 Mar 2026 17:13:11 +0100 Subject: [PATCH 227/230] revert accidentally commited experiment --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8ffc8dd..92e19d3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1344,10 +1344,10 @@ private boolean shouldAutoFlush() { if (autoFlushBytes > 0 && getPendingBytes() >= autoFlushBytes) { return true; } -// if (autoFlushIntervalNanos > 0) { -// long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; -// return ageNanos >= autoFlushIntervalNanos; -// } + if (autoFlushIntervalNanos > 0) { + long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; + return ageNanos >= autoFlushIntervalNanos; + } return false; } From 6d029fe4b4d55da5b26ddc43aaf75743f80968de Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Sat, 21 Mar 2026 11:53:22 +0100 Subject: [PATCH 228/230] Encode null presence inline in column data Replace the nullable type-code flag (0x80 high bit) with an inline null-count byte at the start of each column's data. QwpColumnWriter.writeNullHeader() now writes a single byte (0 = no nulls, 1 = has nulls) followed by the bitmap only when nulls are present. QwpColumnDef no longer stores or OR's a nullable flag into the type code. The hasNullBitmap field and the 3-argument constructor are removed. QwpSchemaHash hashes only the base type code. QwpTableBuffer.ColumnBuffer defers null bitmap expansion to addNull() calls instead of checking capacity on every row. Non-null addXxx() methods no longer touch the bitmap at all. The bitmap is tail-expanded to the full row count at serialization time in writeNullHeader(). Safety fixes: isNull() returns false for indices beyond bitmap capacity, truncateTo() and clearToEmptyFast() clamp to the allocated bitmap size. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/QwpColumnWriter.java | 17 +++---- .../cutlass/qwp/protocol/QwpColumnDef.java | 44 +++---------------- .../cutlass/qwp/protocol/QwpConstants.java | 9 ++-- .../cutlass/qwp/protocol/QwpSchemaHash.java | 5 +-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 44 +++++-------------- .../cutlass/qwp/client/QwpUdpSenderTest.java | 11 ++++- .../qwp/protocol/QwpColumnDefTest.java | 13 +++--- .../qwp/protocol/QwpConstantsTest.java | 9 ++-- .../qwp/protocol/QwpSchemaHashTest.java | 8 ++-- 9 files changed, 52 insertions(+), 108 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index 088e8e6..e07bd7f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -58,9 +58,7 @@ private void encodeColumn( ) { long dataAddr = col.getDataAddress(); - if (colDef.hasNullBitmap()) { - writeNullBitmap(col, rowCount); - } + writeNullHeader(col, rowCount, rowCount - valueCount); switch (col.getType()) { case TYPE_BOOLEAN: @@ -281,16 +279,15 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { } } - private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) { - long nullAddr = col.getNullBitmapAddress(); - if (nullAddr != 0) { + private void writeNullHeader(QwpTableBuffer.ColumnBuffer col, int rowCount, int nullCount) { + if (nullCount > 0) { + buffer.putByte((byte) 1); + col.ensureNullBitmapCapacity(rowCount); + long nullAddr = col.getNullBitmapAddress(); int bitmapSize = (rowCount + 7) / 8; buffer.putBlockOfBytes(nullAddr, bitmapSize); } else { - int bitmapSize = (rowCount + 7) / 8; - for (int i = 0; i < bitmapSize; i++) { - buffer.putByte((byte) 0); - } + buffer.putByte((byte) 0); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index 0f887e4..eef6bf0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -34,33 +34,17 @@ */ public final class QwpColumnDef { private final String name; - private final boolean hasNullBitmap; private final byte typeCode; /** * Creates a column definition. * * @param name the column name (UTF-8) - * @param typeCode the QWP v1 type code (0x01-0x0F, optionally OR'd with 0x80 for null bitmap) + * @param typeCode the QWP v1 type code (0x01-0x16) */ public QwpColumnDef(String name, byte typeCode) { this.name = name; - // Extract null bitmap flag (high bit) and base type - this.hasNullBitmap = (typeCode & 0x80) != 0; - this.typeCode = (byte) (typeCode & 0x7F); - } - - /** - * Creates a column definition with explicit null bitmap flag. - * - * @param name the column name - * @param typeCode the base type code (0x01-0x0F) - * @param hasNullBitmap whether the column has a null bitmap - */ - public QwpColumnDef(String name, byte typeCode, boolean hasNullBitmap) { - this.name = name; - this.typeCode = (byte) (typeCode & 0x7F); - this.hasNullBitmap = hasNullBitmap; + this.typeCode = typeCode; } @Override @@ -69,7 +53,6 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; QwpColumnDef that = (QwpColumnDef) o; return typeCode == that.typeCode && - hasNullBitmap == that.hasNullBitmap && name.equals(that.name); } @@ -81,9 +64,9 @@ public String getName() { } /** - * Gets the base type code (without null bitmap flag). + * Gets the base type code. * - * @return type code 0x01-0x0F + * @return type code 0x01-0x16 */ public byte getTypeCode() { return typeCode; @@ -97,37 +80,24 @@ public String getTypeName() { } /** - * Gets the wire type code (with null bitmap flag if applicable). + * Gets the wire type code. * * @return type code as sent on wire */ public byte getWireTypeCode() { - return hasNullBitmap ? (byte) (typeCode | 0x80) : typeCode; + return typeCode; } @Override public int hashCode() { int result = name.hashCode(); result = 31 * result + typeCode; - result = 31 * result + (hasNullBitmap ? 1 : 0); return result; } - /** - * Returns true if this column has a null bitmap. - */ - public boolean hasNullBitmap() { - return hasNullBitmap; - } - @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(name).append(':').append(getTypeName()); - if (hasNullBitmap) { - sb.append('?'); - } - return sb.toString(); + return name + ':' + getTypeName(); } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index d0f31ce..b8abee6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -269,7 +269,7 @@ private QwpConstants() { * @return size in bytes, 0 for bit-packed (BOOLEAN), or -1 for variable-width types */ public static int getFixedTypeSize(byte typeCode) { - int code = typeCode & TYPE_MASK; + int code = typeCode; switch (code) { case TYPE_BOOLEAN: return 0; // Special: bit-packed @@ -308,8 +308,7 @@ public static int getFixedTypeSize(byte typeCode) { * @return type name */ public static String getTypeName(byte typeCode) { - int code = typeCode & TYPE_MASK; - boolean hasNullBitmap = (typeCode & TYPE_NULLABLE_FLAG) != 0; + int code = typeCode; String name; switch (code) { case TYPE_BOOLEAN: @@ -382,7 +381,7 @@ public static String getTypeName(byte typeCode) { name = "UNKNOWN(" + code + ")"; break; } - return hasNullBitmap ? name + "?" : name; + return name; } /** @@ -392,7 +391,7 @@ public static String getTypeName(byte typeCode) { * @return true if fixed-width */ public static boolean isFixedWidthType(byte typeCode) { - int code = typeCode & TYPE_MASK; + int code = typeCode; return code == TYPE_BOOLEAN || code == TYPE_BYTE || code == TYPE_SHORT || diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index d5decc4..8c7db9c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -152,9 +152,8 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList= nullBufCapRows) { return false; } long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; @@ -1422,7 +1405,7 @@ public void truncateTo(int newSize) { } } // Clear null bits for truncated rows - for (int i = newSize; i < size; i++) { + for (int i = newSize; i < Math.min(size, nullBufCapRows); i++) { long longAddr = nullBufPtr + ((long) (i >>> 6)) * 8; int bitIndex = i & 63; long current = Unsafe.getUnsafe().getLong(longAddr); @@ -1617,7 +1600,8 @@ private void clearToEmptyFast() { int sizeBefore = size; clearValuePayload(); if (nullBufPtr != 0 && sizeBefore > 0) { - long usedLongs = ((long) sizeBefore + 63) >>> 6; + int rowsToClear = Math.min(sizeBefore, nullBufCapRows); + long usedLongs = ((long) rowsToClear + 63) >>> 6; Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); } size = 0; @@ -1650,7 +1634,8 @@ private void compactNullBitmap(int sourceRow) { } boolean retainedNull = isNull(sourceRow); - long usedLongs = ((long) size + 63) >>> 6; + int rowsToClear = Math.min(size, nullBufCapRows); + long usedLongs = ((long) rowsToClear + 63) >>> 6; Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0); if (retainedNull) { Unsafe.getUnsafe().putLong(nullBufPtr, 1L); @@ -1664,11 +1649,6 @@ private void ensureArrayCapacity(int nDims, int dataElements) { arrayDims = Arrays.copyOf(arrayDims, arrayDims.length * 2); } - // Ensure null bitmap capacity - if (useNullBitmap) { - ensureNullBitmapCapacity(); - } - // Ensure shape array capacity int requiredShapeCapacity = arrayShapeOffset + nDims; if (arrayShapes == null) { @@ -1694,11 +1674,11 @@ private void ensureArrayCapacity(int nDims, int dataElements) { } } - private void ensureNullBitmapCapacity() { - if (nullBufPtr == 0 || nullBufCapRows > size) { + public void ensureNullBitmapCapacity(int minRows) { + if (nullBufPtr == 0 || nullBufCapRows >= minRows) { return; } - int newCapRows = Math.max(nullBufCapRows * 2, ((size + 64) >>> 6) << 6); + int newCapRows = Math.max(nullBufCapRows * 2, ((minRows + 63) >>> 6) << 6); long newSizeBytes = (long) newCapRows >>> 3; long oldSizeBytes = (long) nullBufCapRows >>> 3; nullBufPtr = Unsafe.realloc(nullBufPtr, oldSizeBytes, newSizeBytes, MemoryTag.NATIVE_ILP_RSS); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 5a889b2..d675401 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -2164,8 +2164,15 @@ private void decodeBooleans(Object[] values, boolean[] nulls, int valueCount) { } private ColumnValues decodeColumn(QwpColumnDef def, int rowCount) { - boolean[] nulls = def.hasNullBitmap() ? reader.readNullBitmap(rowCount) : new boolean[rowCount]; - int valueCount = rowCount - countNulls(nulls); + boolean hasNullBitmap = reader.readByte() != 0; + boolean[] nulls = hasNullBitmap ? reader.readNullBitmap(rowCount) : new boolean[rowCount]; + int nullCount = 0; + for (boolean isNull : nulls) { + if (isNull) { + nullCount++; + } + } + int valueCount = rowCount - nullCount; Object[] values = new Object[rowCount]; switch (def.getTypeCode()) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java index 7859e0f..b69c646 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java @@ -29,7 +29,6 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; public class QwpColumnDefTest { @@ -74,14 +73,12 @@ public void testValidateCharType() { assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode()); } - @Test - public void testValidateNullableCharType() { - // TYPE_CHAR with nullable flag must also pass - byte nullableChar = (byte) (QwpConstants.TYPE_CHAR | QwpConstants.TYPE_NULLABLE_FLAG); - QwpColumnDef col = new QwpColumnDef("ch", nullableChar); + @Test(expected = IllegalArgumentException.class) + public void testValidateRejectsHighBit() { + // The high bit is not a valid part of the type code + byte badType = (byte) (QwpConstants.TYPE_CHAR | 0x80); + QwpColumnDef col = new QwpColumnDef("ch", badType); col.validate(); - assertTrue(col.hasNullBitmap()); - assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode()); } @Test(expected = IllegalArgumentException.class) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java index c024457..5f63f38 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -91,12 +91,9 @@ public void testGetTypeName() { Assert.assertEquals("DECIMAL256", QwpConstants.getTypeName(TYPE_DECIMAL256)); Assert.assertEquals("CHAR", QwpConstants.getTypeName(TYPE_CHAR)); - // Test nullable types - byte nullableInt = (byte) (TYPE_INT | TYPE_NULLABLE_FLAG); - Assert.assertEquals("INT?", QwpConstants.getTypeName(nullableInt)); - - byte nullableString = (byte) (TYPE_STRING | TYPE_NULLABLE_FLAG); - Assert.assertEquals("STRING?", QwpConstants.getTypeName(nullableString)); + // Type codes with high bit set are unknown — the high bit is not used + byte badInt = (byte) (TYPE_INT | 0x80); + Assert.assertTrue(QwpConstants.getTypeName(badInt).startsWith("UNKNOWN")); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java index 0a8326a..84b72fe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java @@ -189,18 +189,16 @@ public void testNameAffectsHash() { } @Test - public void testNullableFlagAffectsHash() { + public void testDifferentTypeCodesProduceDifferentHashes() { String[] names = {"value"}; - // Non-nullable byte[] types1 = {0x05}; // LONG - // Nullable (high bit set) - byte[] types2 = {(byte) 0x85}; // LONG | 0x80 + byte[] types2 = {0x06}; // DOUBLE long hash1 = QwpSchemaHash.computeSchemaHash(names, types1); long hash2 = QwpSchemaHash.computeSchemaHash(names, types2); - Assert.assertNotEquals("Nullable flag should affect hash", hash1, hash2); + Assert.assertNotEquals("Different types should produce different hashes", hash1, hash2); } @Test From 504721dba10d637f4e5d13e959579b2e7a39647f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Sat, 21 Mar 2026 12:39:59 +0100 Subject: [PATCH 229/230] Remove dead TYPE_MASK and TYPE_NULLABLE_FLAG The nullable-flag-in-type-code mechanism was replaced by inline null count encoding. Since the protocol was never released, remove the dead constants and their tests entirely rather than deprecating them. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/protocol/QwpConstants.java | 8 -------- .../test/cutlass/qwp/protocol/QwpConstantsTest.java | 10 ---------- 2 files changed, 18 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index b8abee6..c8685e4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -209,14 +209,6 @@ public final class QwpConstants { * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] */ public static final byte TYPE_LONG_ARRAY = 0x12; - /** - * Mask for type code without nullable flag. - */ - public static final byte TYPE_MASK = 0x7F; - /** - * High bit indicating nullable column. - */ - public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80; /** * Column type: SHORT (int16, little-endian). */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java index 5f63f38..9bb14c0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -172,16 +172,6 @@ public void testMagicBytesValue() { Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 24) & 0xFF), expected[3]); } - @Test - public void testNullableFlag() { - Assert.assertEquals((byte) 0x80, TYPE_NULLABLE_FLAG); - Assert.assertEquals(0x7F, TYPE_MASK); - - // Test nullable type extraction - byte nullableInt = (byte) (TYPE_INT | TYPE_NULLABLE_FLAG); - Assert.assertEquals(TYPE_INT, nullableInt & TYPE_MASK); - } - @Test public void testSchemaModes() { Assert.assertEquals(0x00, SCHEMA_MODE_FULL); From 50e89f84d5b5be6e29e37fe363fff73aaa50db1b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Sun, 22 Mar 2026 11:01:19 +0100 Subject: [PATCH 230/230] Merge fallout --- .../io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java | 3 --- .../test/cutlass/qwp/client/QwpWebSocketEncoderTest.java | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index e151674..9d01120 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -1132,9 +1132,6 @@ public void addSymbolWithGlobalId(CharSequence value, int globalId) { addNull(); return; } - if (auxBuffer == null) { - auxBuffer = new OffHeapAppendMemory(64); - } if (!storeGlobalSymbolIdsOnly) { if (symbolList != null && symbolList.size() > 0) { int localIdx = getOrAddLocalSymbol(value); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index a118b0e..7ab65bf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -994,6 +994,7 @@ public void testEncodeWithDeltaDict_readsGlobalIdsFromDataBuffer() throws Except Assert.assertEquals(SCHEMA_MODE_FULL, cursor.readByte()); Assert.assertEquals("ticker", cursor.readString()); Assert.assertEquals(TYPE_SYMBOL, cursor.readByte()); + Assert.assertEquals(0, cursor.readByte()); // no nulls Assert.assertEquals(5, cursor.readVarint()); Assert.assertEquals(7, cursor.readVarint()); }